bes  Updated for version 3.17.0
BESDapResponseBuilder.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 #include <signal.h>
28 #include <unistd.h>
29 #include <sys/stat.h>
30 
31 #ifdef HAVE_UUID_UUID_H
32 #include <uuid/uuid.h> // used to build CID header value for data ddx
33 #elif defined(HAVE_UUID_H)
34 #include <uuid.h>
35 #else
36 #error "Could not find UUID library header"
37 #endif
38 
39 
40 #ifndef WIN32
41 #include <sys/wait.h>
42 #else
43 #include <io.h>
44 #include <fcntl.h>
45 #include <process.h>
46 #endif
47 
48 #include <iostream>
49 #include <string>
50 #include <sstream>
51 #include <fstream>
52 
53 #include <cstring>
54 #include <ctime>
55 
56 //#define DODS_DEBUG
57 #define CLEAR_LOCAL_DATA
58 #undef FUNCTION_CACHING
59 #undef USE_LOCAL_TIMEOUT_SCHEME
60 
61 #include <DAS.h>
62 #include <DDS.h>
63 #include <Structure.h>
64 #include <ConstraintEvaluator.h>
65 #include <DDXParserSAX2.h>
66 #include <Ancillary.h>
67 #include <XDRStreamMarshaller.h>
68 #include <XDRFileUnMarshaller.h>
69 
70 #include <DMR.h>
71 #include <D4Group.h>
72 #include <XMLWriter.h>
73 #include <D4AsyncUtil.h>
74 #include <D4StreamMarshaller.h>
75 #include <chunked_ostream.h>
76 #include <chunked_istream.h>
77 #include <D4ConstraintEvaluator.h>
78 #include <D4FunctionEvaluator.h>
79 #include <D4BaseTypeFactory.h>
80 
81 #include <ServerFunctionsList.h>
82 
83 #include <mime_util.h> // for last_modified_time() and rfc_822_date()
84 #include <escaping.h>
85 #include <util.h>
86 #if USE_LOCAL_TIMEOUT_SCHEME
87 #ifndef WIN32
88 #include <SignalHandler.h>
89 #include <EventHandler.h>
90 #include <AlarmHandler.h>
91 #endif
92 #endif
93 
94 #include "TheBESKeys.h"
95 #include "BESDapResponseBuilder.h"
96 #include "BESContextManager.h"
97 #include "BESDapResponseCache.h"
98 #include "BESStoredDapResultCache.h"
99 #include "BESUtil.h"
100 #include "BESDebug.h"
101 #include "BESStopWatch.h"
102 
103 using namespace std;
104 using namespace libdap;
105 
106 const string CRLF = "\r\n"; // Change here, expr-test.cc
107 const string BES_KEY_TIMEOUT_CANCEL = "BES.CancelTimeoutOnSend";
108 
114 {
115  bool found = false;
116  string cancel_timeout_on_send = "";
117  TheBESKeys::TheKeys()->get_value(BES_KEY_TIMEOUT_CANCEL, cancel_timeout_on_send, found);
118  if (found && !cancel_timeout_on_send.empty()) {
119  // The default value is false.
120  downcase(cancel_timeout_on_send);
121  if (cancel_timeout_on_send == "yes" || cancel_timeout_on_send == "true")
122  d_cancel_timeout_on_send = true;
123  }
124 }
125 
126 BESDapResponseBuilder::~BESDapResponseBuilder()
127 {
128 #if USE_LOCAL_TIMEOUT_SCHEME
129  // If an alarm was registered, delete it. The register code in SignalHandler
130  // always deletes the old alarm handler object, so only the one returned by
131  // remove_handler needs to be deleted at this point.
132  delete dynamic_cast<AlarmHandler*>(SignalHandler::instance()->remove_handler(SIGALRM));
133 #endif
134 }
135 
143 {
144  return d_dap2ce;
145 }
146 
158 {
159  d_dap2ce = www2id(_ce, "%", "%20");
160 }
161 
166 {
167  return d_dap4ce;
168 }
169 
181 {
182  d_dap4ce = www2id(_ce, "%", "%20");
183 }
184 
189 {
190  return d_dap4function;
191 }
192 
205 {
206  d_dap4function = www2id(_func, "%", "%20");
207 }
208 
209 std::string BESDapResponseBuilder::get_store_result() const
210 {
211  return d_store_result;
212 }
213 
214 void BESDapResponseBuilder::set_store_result(std::string _sr)
215 {
216  d_store_result = _sr;
217  BESDEBUG("dap", "BESDapResponseBuilder::set_store_result() - store_result: " << _sr << endl);
218 }
219 
220 std::string BESDapResponseBuilder::get_async_accepted() const
221 {
222  return d_async_accepted;
223 }
224 
225 void BESDapResponseBuilder::set_async_accepted(std::string _aa)
226 {
227  d_async_accepted = _aa;
228  BESDEBUG("dap", "BESDapResponseBuilder::set_async_accepted() - async_accepted: " << _aa << endl);
229 }
230 
240 {
241  return d_dataset;
242 }
243 
255 {
256  d_dataset = www2id(ds, "%", "%20");
257 }
258 
265 {
266  d_timeout = t;
267 }
268 
271 {
272  return d_timeout;
273 }
274 
281 void
283 {
284 #if USE_LOCAL_TIMEOUT_SCHEME
285 #ifndef WIN32
286  alarm(d_timeout);
287 #endif
288 #endif
289 }
290 
296 void
298 {
299 #if USE_LOCAL_TIMEOUT_SCHEME
300 #ifndef WIN32
301  alarm(0);
302 #endif
303 #endif
304 }
305 
321 {
322  if (d_cancel_timeout_on_send)
323  alarm(0);
324 }
325 
335 {
336 #if USE_LOCAL_TIMEOUT_SCHEME
337 #ifndef WIN32
338  SignalHandler *sh = SignalHandler::instance();
339  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
340  delete old_eh;
341 #endif
342 #endif
343 }
344 
345 
352 {
353 #if USE_LOCAL_TIMEOUT_SCHEME
354 #ifndef WIN32
355  if (d_timeout > 0) {
356  SignalHandler *sh = SignalHandler::instance();
357  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
358  delete old_eh;
359  alarm(d_timeout);
360  }
361 #endif
362 #endif
363 }
364 
369 {
370 #if USE_LOCAL_TIMEOUT_SCHEME
371  alarm(0);
372 #endif
373 }
374 
375 static string::size_type find_closing_paren(const string &ce, string::size_type pos)
376 {
377  // Iterate over the string finding all ( or ) characters until the matching ) is found.
378  // For each ( found, increment count. When a ) is found and count is zero, it is the
379  // matching closing paren, otherwise, decrement count and keep looking.
380  int count = 1;
381  do {
382  pos = ce.find_first_of("()", pos + 1);
383  if (pos == string::npos)
384  throw Error(malformed_expr, "Expected to find a matching closing parenthesis in " + ce);
385 
386  if (ce[pos] == '(')
387  ++count;
388  else
389  --count; // must be ')'
390 
391  } while (count > 0);
392 
393  return pos;
394 }
395 
402 void BESDapResponseBuilder::split_ce(ConstraintEvaluator &eval, const string &expr)
403 {
404  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - source expression: " << expr << endl);
405 
406  string ce;
407  if (!expr.empty())
408  ce = expr;
409  else
410  ce = d_dap2ce;
411 
412  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - ce: " << ce << endl);
413 
414  string btp_function_ce = "";
415  string::size_type pos = 0;
416 
417  // This hack assumes that the functions are listed first. Look for the first
418  // open paren and the last closing paren to accommodate nested function calls
419  string::size_type first_paren = ce.find("(", pos);
420  string::size_type closing_paren = string::npos;
421  if (first_paren != string::npos) closing_paren = find_closing_paren(ce, first_paren); //ce.find(")", pos);
422 
423  while (first_paren != string::npos && closing_paren != string::npos) {
424  // Maybe a BTP function; get the name of the potential function
425  string name = ce.substr(pos, first_paren - pos);
426 
427  // is this a BTP function
428  btp_func f;
429  if (eval.find_function(name, &f)) {
430  // Found a BTP function
431  if (!btp_function_ce.empty()) btp_function_ce += ",";
432  btp_function_ce += ce.substr(pos, closing_paren + 1 - pos);
433  ce.erase(pos, closing_paren + 1 - pos);
434  if (ce[pos] == ',') ce.erase(pos, 1);
435  }
436  else {
437  pos = closing_paren + 1;
438  // exception?
439  if (pos < ce.length() && ce.at(pos) == ',') ++pos;
440  }
441 
442  first_paren = ce.find("(", pos);
443  closing_paren = ce.find(")", pos);
444  }
445 
446  d_dap2ce = ce;
447  d_btp_func_ce = btp_function_ce;
448 
449  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - Modified constraint: " << d_dap2ce << endl);
450  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - BTP Function part: " << btp_function_ce << endl);
451  BESDEBUG("dap", "BESDapResponseBuilder::split_ce() - END" << endl);
452 }
453 
468 void BESDapResponseBuilder::send_das(ostream &out, DAS &das, bool with_mime_headers) const
469 {
470  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
471 
472  das.print(out);
473 
474  out << flush;
475 }
476 
494 void BESDapResponseBuilder::send_das(ostream &out, DDS &dds, ConstraintEvaluator &eval, bool constrained,
495  bool with_mime_headers)
496 {
497 #if USE_LOCAL_TIMEOUT_SCHEME
498  // Set up the alarm.
499  establish_timeout(out);
500  dds.set_timeout(d_timeout);
501 #endif
502  if (!constrained) {
503  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
504 
505  conditional_timeout_cancel();
506 
507  dds.print_das(out);
508  out << flush;
509 
510  return;
511  }
512 
513  split_ce(eval);
514 
515  // If there are functions, parse them and eval.
516  // Use that DDS and parse the non-function ce
517  // Serialize using the second ce and the second dds
518  if (!d_btp_func_ce.empty()) {
519  DDS *fdds = 0;
520  string cache_token = "";
521  ConstraintEvaluator func_eval;
523 
524  if (responseCache) {
525  fdds = responseCache->cache_dataset(dds, d_btp_func_ce, this, &func_eval, cache_token);
526  }
527  else {
528  func_eval.parse_constraint(d_btp_func_ce, dds);
529  fdds = func_eval.eval_function_clauses(dds);
530  }
531 
532  if (with_mime_headers)
533  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
534 
535  conditional_timeout_cancel();
536 
537 
538  fdds->print_das(out);
539 
540  if (responseCache)
541  responseCache->unlock_and_close(cache_token);
542 
543  delete fdds;
544  }
545  else {
546  eval.parse_constraint(d_dap2ce, dds); // Throws Error if the ce doesn't parse.
547 
548  if (with_mime_headers)
549  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
550 
551  conditional_timeout_cancel();
552 
553 
554  dds.print_das(out);
555  }
556 
557  out << flush;
558 }
559 
564 static bool ends_with (const string &full_string, const string &ending) {
565  if (full_string.length() >= ending.length()) {
566  return (0 == full_string.compare (full_string.length() - ending.length(), ending.length(), ending));
567  } else {
568  return false;
569  }
570 }
571 
600 static DDS *promote_function_output_structure(DDS *fdds)
601 {
602  // Look in the top level of the DDS for a promotable member - i.e. a member
603  // variable that is a collection and whose name ends with "_unwrap"
604  bool found_promotable_member = false;
605  for (DDS::Vars_citer di = fdds->var_begin(), de = fdds->var_end(); di != de && !found_promotable_member; ++di) {
606  Structure *collection = dynamic_cast<Structure *>(*di);
607  if (collection && ends_with(collection->name(), "_unwrap")) {
608  found_promotable_member = true;
609  }
610  }
611 
612  // If we found one or more promotable member variables, promote them.
613  if (found_promotable_member) {
614 
615  // Dump pointers to the values here temporarily... If we had methods in libdap
616  // that could be used to access the underlying erase() and insert() methods, we
617  // could skip the (maybe expensive) copy operations I use below. What we would
618  // need are ways to delete a Structure/Constructor without calling delete on its
619  // fields and ways to call vector::erase() and vector::insert(). Some of this
620  // exists, but it's not quite enough.
621 
622  DDS *temp_dds = new DDS(fdds->get_factory(), fdds->get_dataset_name(), fdds->get_dap_version());
623 
624  for (DDS::Vars_citer di = fdds->var_begin(), de = fdds->var_end(); di != de; ++di) {
625  Structure *collection = dynamic_cast<Structure *>(*di);
626  if (collection && ends_with(collection->name(), "_unwrap")) {
627  // So we're going to 'flatten this structure' and return its fields
628  Structure::Vars_iter vi;
629  for (vi =collection->var_begin(); vi != collection->var_end(); ++vi) {
630  temp_dds->add_var(*vi); // better to use add_var_nocopy(*vi); need to modify libdap?
631  }
632  }
633  else {
634  temp_dds->add_var(*di);
635  }
636  }
637 
638  delete fdds;
639  return temp_dds;
640  }
641  else {
642  // Otherwise do nothing to alter the DDS
643  return fdds;
644  }
645 }
646 
665 void BESDapResponseBuilder::send_dds(ostream &out, DDS &dds, ConstraintEvaluator &eval, bool constrained,
666  bool with_mime_headers)
667 {
668  if (!constrained) {
669  if (with_mime_headers)
670  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
671 
672  conditional_timeout_cancel();
673 
674 
675  dds.print(out);
676  out << flush;
677  return;
678  }
679 
680 #if USE_LOCAL_TIMEOUT_SCHEME
681  // Set up the alarm.
682  establish_timeout(out);
683  dds.set_timeout(d_timeout);
684 #endif
685 
686  // Split constraint into two halves
687  split_ce(eval);
688 
689  // If there are functions, parse them and eval.
690  // Use that DDS and parse the non-function ce
691  // Serialize using the second ce and the second dds
692  if (!d_btp_func_ce.empty()) {
693  string cache_token = "";
694  DDS *fdds = 0;
695  ConstraintEvaluator func_eval;
696 
698 
699  if (responseCache) {
700  fdds = responseCache->cache_dataset(dds, d_btp_func_ce, this, &func_eval, cache_token);
701  }
702  else {
703  func_eval.parse_constraint(d_btp_func_ce, dds);
704  fdds = func_eval.eval_function_clauses(dds);
705  }
706 
707  // Server functions might mark variables to use their read()
708  // methods. Clear that so the CE in d_dap2ce will control what is
709  // sent. If that is empty (there was only a function call) all
710  // of the variables in the intermediate DDS (i.e., the function
711  // result) will be sent.
712  fdds->mark_all(false);
713 
714  eval.parse_constraint(d_dap2ce, *fdds);
715 
716  if (with_mime_headers)
717  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
718 
719  fdds = promote_function_output_structure(/*&*/fdds);
720 
721  conditional_timeout_cancel();
722 
723 
724  fdds->print_constrained(out);
725 
726  if (responseCache)
727  responseCache->unlock_and_close(cache_token);
728 
729  delete fdds;
730  }
731  else {
732  eval.parse_constraint(d_dap2ce, dds); // Throws Error if the ce doesn't parse.
733 
734  if (with_mime_headers)
735  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
736 
737  conditional_timeout_cancel();
738 
739 
740  dds.print_constrained(out);
741  }
742 
743  out << flush;
744 }
745 
760 bool BESDapResponseBuilder::store_dap2_result(ostream &out, DDS &dds, ConstraintEvaluator &eval)
761 {
762  if (get_store_result().empty()) return false;
763 
764  string serviceUrl = get_store_result();
765 
766  XMLWriter xmlWrtr;
767  D4AsyncUtil d4au;
768 
769  // FIXME Keys should be read in initialize(). Also, I think the D4AsyncUtil should
770  // be removed from libdap - it is much more about how the BES processes these kinds
771  // of operations. Change this when working on the response caching for ODSIP. But...
772  // do we really need to put the style sheet in the bes.conf file? Should it be baked
773  // into the code (because we don't want people to change it)?
774  bool found;
775  string *stylesheet_ref = 0, ss_ref_value;
776  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
777  if (found && ss_ref_value.length() > 0) {
778  stylesheet_ref = &ss_ref_value;
779  }
780 
782  if (resultCache == NULL) {
783 
789  string msg = "The Stored Result request cannot be serviced. ";
790  msg += "Unable to acquire StoredResultCache instance. ";
791  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
792 
793  BESDEBUG("dap", "[WARNING] " << msg << endl);
794 
795  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
796  out << xmlWrtr.get_doc();
797  out << flush;
798 
799  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - Sent AsyncRequestRejected" << endl);
800  }
801  else if (get_async_accepted().length() != 0) {
802 
806  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - serviceUrl="<< serviceUrl << endl);
807 
809  string storedResultId = "";
810  storedResultId = resultCache->store_dap2_result(dds, get_ce(), this, &eval);
811 
812  BESDEBUG("dap",
813  "BESDapResponseBuilder::store_dap2_result() - storedResultId='"<< storedResultId << "'" << endl);
814 
815  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
816  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - targetURL='"<< targetURL << "'" << endl);
817 
818  XMLWriter xmlWrtr;
819  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
820  out << xmlWrtr.get_doc();
821  out << flush;
822 
823  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - sent DAP4 AsyncAccepted response" << endl);
824  }
825  else {
830  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
831  out << xmlWrtr.get_doc();
832  out << flush;
833 
834  BESDEBUG("dap", "BESDapResponseBuilder::store_dap2_result() - sent DAP4 AsyncRequired response" << endl);
835  }
836 
837  return true;
838 
839 }
840 
844 void BESDapResponseBuilder::serialize_dap2_data_dds(ostream &out, DDS &dds, ConstraintEvaluator &eval, bool ce_eval)
845 {
846  BESStopWatch sw;
847  if (BESISDEBUG(TIMING_LOG)) sw.start("BESDapResponseBuilder::serialize_dap2_data_dds", "");
848 
849  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap2_data_dds() - BEGIN" << endl);
850 
851  dds.print_constrained(out);
852  out << "Data:\n";
853  out << flush;
854 
855  XDRStreamMarshaller m(out);
856 
857  // This only has an effect when the timeout in BESInterface::execute_request()
858  // is set. Otherwise it does nothing.
859  conditional_timeout_cancel();
860 
861 
862  // Send all variables in the current projection (send_p())
863  for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++) {
864  if ((*i)->send_p()) {
865  (*i)->serialize(eval, dds, m, ce_eval);
866 #ifdef CLEAR_LOCAL_DATA
867  (*i)->clear_local_data();
868 #endif
869  }
870  }
871 
872  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap2_data_dds() - END" << endl);
873 }
874 
883 void BESDapResponseBuilder::serialize_dap2_data_ddx(ostream &out, DDS &dds, ConstraintEvaluator &eval,
884  const string &boundary, const string &start, bool ce_eval)
885 {
886  BESDEBUG("dap", __PRETTY_FUNCTION__ << " BEGIN" << endl);
887 
888  // Write the MPM headers for the DDX (text/xml) part of the response
889  libdap::set_mime_ddx_boundary(out, boundary, start, dods_ddx, x_plain);
890 
891  // Make cid
892  uuid_t uu;
893  uuid_generate(uu);
894  char uuid[37];
895  uuid_unparse(uu, &uuid[0]);
896  char domain[256];
897  if (getdomainname(domain, 255) != 0 || strlen(domain) == 0) strncpy(domain, "opendap.org", 255);
898 
899  string cid = string(&uuid[0]) + "@" + string(&domain[0]);
900 
901  // Send constrained DDX with a data blob reference.
902  // FIXME Comment CID passed but ignored jhrg 10/20/15
903  dds.print_xml_writer(out, true, cid);
904 
905  // write the data part mime headers here
906  set_mime_data_boundary(out, boundary, cid, dods_data_ddx /* old value dap4_data*/, x_plain);
907 
908  XDRStreamMarshaller m(out);
909 
910  conditional_timeout_cancel();
911 
912 
913  // Send all variables in the current projection (send_p()).
914  for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); i++) {
915  if ((*i)->send_p()) {
916  (*i)->serialize(eval, dds, m, ce_eval);
917 #ifdef CLEAR_LOCAL_DATA
918  (*i)->clear_local_data();
919 #endif
920  }
921  }
922 
923  BESDEBUG("dap", __PRETTY_FUNCTION__ << " END" << endl);
924 }
925 
942 void BESDapResponseBuilder::send_dap2_data(ostream &data_stream, DDS &dds, ConstraintEvaluator &eval,
943  bool with_mime_headers)
944 {
945  BESDEBUG("dap", "BESDapResponseBuilder::send_dap2_data() - BEGIN"<< endl);
946 
947 #if USE_LOCAL_TIMEOUT_SCHEME
948  // Set up the alarm.
949  establish_timeout(data_stream);
950  dds.set_timeout(d_timeout);
951 #endif
952 
953  // Split constraint into two halves
954  split_ce(eval);
955 
956  // If there are functions, parse them and eval.
957  // Use that DDS and parse the non-function ce
958  // Serialize using the second ce and the second dds
959  if (!get_btp_func_ce().empty()) {
960  BESDEBUG("dap",
961  "BESDapResponseBuilder::send_dap2_data() - Found function(s) in CE: " << get_btp_func_ce() << endl);
962 
963  string cache_token = "";
964  DDS *fdds = 0;
965  // Define a local ce evaluator so that the clause from the function parse
966  // won't get treated like selection clauses later on when serialize is called
967  // on the DDS (fdds)
968  ConstraintEvaluator func_eval;
969  BESDapResponseCache *responseCache = 0;
970 
971 #if FUNCTION_CACHING
972  responseCache = BESDapResponseCache::get_instance();
973 #endif
974 
975  if (responseCache) {
976  BESDEBUG("dap",
977  "BESDapResponseBuilder::send_dap2_data() - Using the cache for the server function CE" << endl);
978  fdds = responseCache->cache_dataset(dds, get_btp_func_ce(), this, &func_eval, cache_token);
979  }
980  else {
981  BESDEBUG("dap", "BESDapResponseBuilder::send_dap2_data() - Cache not found; (re)calculating" << endl);
982  func_eval.parse_constraint(get_btp_func_ce(), dds);
983  fdds = func_eval.eval_function_clauses(dds);
984  }
985 
986  // Server functions might mark variables to use their read()
987  // methods. Clear that so the CE in d_dap2ce will control what is
988  // sent. If that is empty (there was only a function call) all
989  // of the variables in the intermediate DDS (i.e., the function
990  // result) will be sent.
991  fdds->mark_all(false);
992 
993  fdds = promote_function_output_structure(fdds);
994 
995  eval.parse_constraint(get_ce(), *fdds);
996 
997  fdds->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
998 
999  if (fdds->get_response_limit() != 0 && fdds->get_request_size(true) > fdds->get_response_limit()) {
1000  string msg = "The Request for " + long_to_string(dds.get_request_size(true) / 1024)
1001  + "KB is too large; requests for this user are limited to "
1002  + long_to_string(dds.get_response_limit() / 1024) + "KB.";
1003  throw Error(msg);
1004  }
1005 
1006  if (with_mime_headers)
1007  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
1008 
1009 #if FUNCTION_CACHING
1010  // This means: if we are not supposed to store the result, then serialize it.
1011  if (!store_dap2_result(data_stream, dds, eval)) {
1012  serialize_dap2_data_dds(data_stream, *fdds, eval, true /* was 'false'. jhrg 3/10/15 */);
1013  }
1014 
1015  if (responseCache)
1016  responseCache->unlock_and_close(cache_token);
1017 #else
1018  serialize_dap2_data_dds(data_stream, *fdds, eval, true /* was 'false'. jhrg 3/10/15 */);
1019 #endif
1020 
1021  delete fdds;
1022  }
1023  else {
1024  BESDEBUG("dap", "BESDapResponseBuilder::send_dap2_data() - Simple constraint" << endl);
1025 
1026  eval.parse_constraint(get_ce(), dds); // Throws Error if the ce doesn't parse.
1027 
1028  dds.tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1029 
1030  if (dds.get_response_limit() != 0 && dds.get_request_size(true) > dds.get_response_limit()) {
1031  string msg = "The Request for " + long_to_string(dds.get_request_size(true) / 1024)
1032  + "KB is too large; requests for this user are limited to "
1033  + long_to_string(dds.get_response_limit() / 1024) + "KB.";
1034  throw Error(msg);
1035  }
1036 
1037  if (with_mime_headers)
1038  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
1039 
1040  // This means: if we are not supposed to store the result, then serialize it.
1041  if (!store_dap2_result(data_stream, dds, eval)) {
1042  serialize_dap2_data_dds(data_stream, dds, eval);
1043  }
1044  }
1045 
1046  data_stream << flush;
1047 
1048  BESDEBUG("dap", "BESDapResponseBuilder::send_dap2_data() - END"<< endl);
1049 
1050 }
1051 
1052 
1066 void BESDapResponseBuilder::send_ddx(ostream &out, DDS &dds, ConstraintEvaluator &eval, bool with_mime_headers)
1067 {
1068  if (d_dap2ce.empty()) {
1069  if (with_mime_headers)
1070  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
1071 
1072  dds.print_xml_writer(out, false /*constrained */, "");
1073  //dds.print(out);
1074  out << flush;
1075  return;
1076  }
1077 
1078 #if USE_LOCAL_TIMEOUT_SCHEME
1079  // Set up the alarm.
1080  establish_timeout(out);
1081  dds.set_timeout(d_timeout);
1082 #endif
1083 
1084  // Split constraint into two halves
1085  split_ce(eval);
1086 
1087  // If there are functions, parse them and eval.
1088  // Use that DDS and parse the non-function ce
1089  // Serialize using the second ce and the second dds
1090  if (!d_btp_func_ce.empty()) {
1091  string cache_token = "";
1092  DDS *fdds = 0;
1093  ConstraintEvaluator func_eval;
1094  BESDapResponseCache *responseCache = 0;
1095 
1096 #if FUNCTION_CACHING
1097  responseCache = BESDapResponseCache::get_instance();
1098 #endif
1099 
1100  if (responseCache) {
1101  fdds = responseCache->cache_dataset(dds, d_btp_func_ce, this, &func_eval, cache_token);
1102  }
1103  else {
1104  func_eval.parse_constraint(d_btp_func_ce, dds);
1105  fdds = func_eval.eval_function_clauses(dds);
1106  }
1107 
1108  // Server functions might mark variables to use their read()
1109  // methods. Clear that so the CE in d_dap2ce will control what is
1110  // sent. If that is empty (there was only a function call) all
1111  // of the variables in the intermediate DDS (i.e., the function
1112  // result) will be sent.
1113  fdds->mark_all(false);
1114 
1115  fdds = promote_function_output_structure(fdds);
1116 
1117  eval.parse_constraint(d_dap2ce, *fdds);
1118 
1119  if (with_mime_headers)
1120  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
1121 
1122  conditional_timeout_cancel();
1123 
1124 
1125  fdds->print_xml_writer(out, true, "");
1126 
1127 #if FUNCTION_CACHING
1128  if (responseCache)
1129  responseCache->unlock_and_close(cache_token);
1130 #endif
1131 
1132  delete fdds;
1133  }
1134  else {
1135  eval.parse_constraint(d_dap2ce, dds); // Throws Error if the ce doesn't parse.
1136 
1137  if (with_mime_headers)
1138  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), dds.get_dap_version());
1139 
1140  conditional_timeout_cancel();
1141 
1142 
1143  // dds.print_constrained(out);
1144  dds.print_xml_writer(out, true, "");
1145  }
1146 
1147  out << flush;
1148 }
1149 
1150 void BESDapResponseBuilder::send_dmr(ostream &out, DMR &dmr, bool with_mime_headers)
1151 {
1152  // If the CE is not empty, parse it. The projections, etc., are set as a side effect.
1153  // If the parser returns false, the expression did not parse. The parser may also
1154  // throw Error
1155  if (!d_dap4ce.empty()) {
1156 
1157  BESDEBUG("dap", "BESDapResponseBuilder::send_dmr() - Parsing DAP4 constraint: '"<< d_dap4ce << "'"<< endl);
1158 
1159  D4ConstraintEvaluator parser(&dmr);
1160  bool parse_ok = parser.parse(d_dap4ce);
1161  if (!parse_ok) throw Error(malformed_expr, "Constraint Expression (" + d_dap4ce + ") failed to parse.");
1162  }
1163  // with an empty CE, send everything. Even though print_dap4() and serialize()
1164  // don't need this, other code may depend on send_p being set. This may change
1165  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1166  else {
1167  dmr.root()->set_send_p(true);
1168  }
1169 
1170  if (with_mime_headers) set_mime_text(out, dap4_dmr, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1171 
1172  conditional_timeout_cancel();
1173 
1174 
1175  XMLWriter xml;
1176  dmr.print_dap4(xml, /*constrained &&*/!d_dap4ce.empty() /* true == constrained */);
1177  out << xml.get_doc() << flush;
1178 }
1179 
1180 void BESDapResponseBuilder::send_dap4_data_using_ce(ostream &out, DMR &dmr, bool with_mime_headers)
1181 {
1182  if (!d_dap4ce.empty()) {
1183  D4ConstraintEvaluator parser(&dmr);
1184  bool parse_ok = parser.parse(d_dap4ce);
1185  if (!parse_ok) throw Error(malformed_expr, "Constraint Expression (" + d_dap4ce + ") failed to parse.");
1186  }
1187  // with an empty CE, send everything. Even though print_dap4() and serialize()
1188  // don't need this, other code may depend on send_p being set. This may change
1189  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1190  else {
1191  dmr.root()->set_send_p(true);
1192  }
1193 
1194  if (dmr.response_limit() != 0 && dmr.request_size(true) > dmr.response_limit()) {
1195  string msg = "The Request for " + long_to_string(dmr.request_size(true) / 1024)
1196  + "MB is too large; requests for this user are limited to " + long_to_string(dmr.response_limit() / 1024)
1197  + "MB.";
1198  throw Error(msg);
1199  }
1200 
1201  if (!store_dap4_result(out, dmr)) {
1202  serialize_dap4_data(out, dmr, with_mime_headers);
1203  }
1204 }
1205 
1206 void BESDapResponseBuilder::send_dap4_data(ostream &out, DMR &dmr, bool with_mime_headers)
1207 {
1208  // If a function was passed in with this request, evaluate it and use that DMR
1209  // for the remainder of this request.
1210  // TODO Add caching for these function invocations
1211  if (!d_dap4function.empty()) {
1212  D4BaseTypeFactory d4_factory;
1213  DMR function_result(&d4_factory, "function_results");
1214 
1215  // Function modules load their functions onto this list. The list is
1216  // part of libdap, not the BES.
1217  if (!ServerFunctionsList::TheList())
1218  throw Error(
1219  "The function expression could not be evaluated because there are no server functions defined on this server");
1220 
1221  D4FunctionEvaluator parser(&dmr, ServerFunctionsList::TheList());
1222  bool parse_ok = parser.parse(d_dap4function);
1223  if (!parse_ok) throw Error("Function Expression (" + d_dap4function + ") failed to parse.");
1224 
1225  parser.eval(&function_result);
1226 
1227  // Now use the results of running the functions for the remainder of the
1228  // send_data operation.
1229  send_dap4_data_using_ce(out, function_result, with_mime_headers);
1230  }
1231  else {
1232  send_dap4_data_using_ce(out, dmr, with_mime_headers);
1233  }
1234 }
1235 
1239 void BESDapResponseBuilder::serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers)
1240 {
1241  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap4_data() - BEGIN" << endl);
1242 
1243  if (with_mime_headers) set_mime_binary(out, dap4_data, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1244 
1245  // Write the DMR
1246  XMLWriter xml;
1247  dmr.print_dap4(xml, !d_dap4ce.empty());
1248 
1249  // now make the chunked output stream; set the size to be at least chunk_size
1250  // but make sure that the whole of the xml plus the CRLF can fit in the first
1251  // chunk. (+2 for the CRLF bytes).
1252  chunked_ostream cos(out, max((unsigned int) CHUNK_SIZE, xml.get_doc_size() + 2));
1253 
1254  conditional_timeout_cancel();
1255 
1256  // using flush means that the DMR and CRLF are in the first chunk.
1257  cos << xml.get_doc() << CRLF << flush;
1258 
1259  // Write the data, chunked with checksums
1260  D4StreamMarshaller m(cos);
1261  dmr.root()->serialize(m, dmr, !d_dap4ce.empty());
1262 #ifdef CLEAR_LOCAL_DATA
1263  dmr.root()->clear_local_data();
1264 #endif
1265  cos << flush;
1266 
1267  BESDEBUG("dap", "BESDapResponseBuilder::serialize_dap4_data() - END" << endl);
1268 }
1269 
1284 bool BESDapResponseBuilder::store_dap4_result(ostream &out, libdap::DMR &dmr)
1285 {
1286  if (get_store_result().length() != 0) {
1287  string serviceUrl = get_store_result();
1288 
1289  D4AsyncUtil d4au;
1290  XMLWriter xmlWrtr;
1291 
1292  // FIXME See above comment for store dap2 result
1293  bool found;
1294  string *stylesheet_ref = 0, ss_ref_value;
1295  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
1296  if (found && ss_ref_value.length() > 0) {
1297  stylesheet_ref = &ss_ref_value;
1298  }
1299 
1301  if (resultCache == NULL) {
1302 
1308  string msg = "The Stored Result request cannot be serviced. ";
1309  msg += "Unable to acquire StoredResultCache instance. ";
1310  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
1311 
1312  BESDEBUG("dap", "[WARNING] " << msg << endl);
1313  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
1314  out << xmlWrtr.get_doc();
1315  out << flush;
1316  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - Sent AsyncRequestRejected" << endl);
1317 
1318  return true;
1319  }
1320 
1321  if (get_async_accepted().length() != 0) {
1322 
1326  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - serviceUrl="<< serviceUrl << endl);
1327 
1328  string storedResultId = "";
1329  storedResultId = resultCache->store_dap4_result(dmr, get_ce(), this);
1330 
1331  BESDEBUG("dap",
1332  "BESDapResponseBuilder::store_dap4_result() - storedResultId='"<< storedResultId << "'" << endl);
1333 
1334  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
1335  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - targetURL='"<< targetURL << "'" << endl);
1336 
1337  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
1338  out << xmlWrtr.get_doc();
1339  out << flush;
1340  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - sent AsyncAccepted" << endl);
1341 
1342  }
1343  else {
1348  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
1349  out << xmlWrtr.get_doc();
1350  out << flush;
1351  BESDEBUG("dap", "BESDapResponseBuilder::store_dap4_result() - sent AsyncAccepted" << endl);
1352  }
1353 
1354  return true;
1355  }
1356 
1357  return false;
1358 }
1359 
virtual std::string get_ce() const
Get the constraint expression.
virtual std::string get_dap4ce() const
Get the DAP4 constraint expression.
virtual void serialize_dap2_data_dds(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, bool ce_eval=true)
static BESDapResponseCache * get_instance()
STL namespace.
virtual void send_dap2_data(std::ostream &data_stream, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, bool with_mime_headers=true)
Transmit data.
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
virtual bool start(string name)
Definition: BESStopWatch.cc:57
virtual std::string get_dap4function() const
Get the DAP4 server side function expression.
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single &#39;/&#39; character.
Definition: BESUtil.cc:747
static BESStoredDapResultCache * get_instance()
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual void remove_timeout() const
void set_timeout(int timeout=0)
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual void set_dap4function(std::string _func)
virtual void send_ddx(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, bool with_mime_headers=true)
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: BESKeys.cc:481
virtual void serialize_dap2_data_ddx(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, const std::string &boundary, const std::string &start, bool ce_eval=true)
virtual void set_ce(std::string _ce)
bool store_dap2_result(ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval)
static BESKeys * TheKeys()
Definition: TheBESKeys.cc:43
virtual void establish_timeout(std::ostream &stream) const
virtual bool store_dap4_result(ostream &out, libdap::DMR &dmr)
virtual std::string get_dataset_name() const
Get the dataset name.
virtual void set_dap4ce(std::string _ce)
virtual void unlock_and_close(const string &target)
virtual string store_dap2_result(libdap::DDS &dds, const std::string &constraint, BESDapResponseBuilder *rb, libdap::ConstraintEvaluator *eval)
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)
virtual void send_dds(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, bool constrained=false, bool with_mime_headers=true)
Transmit a DDS.