bes  Updated for version 3.20.8
DmrppCommon.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of the BES
4 
5 // Copyright (c) 2016 OPeNDAP, Inc.
6 // Author: James Gallagher <jgallagher@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 #include "config.h"
24 
25 #include <string>
26 #include <sstream>
27 #include <vector>
28 #include <iterator>
29 #include <cstdlib>
30 #include <cstring>
31 
32 #include <curl/curl.h>
33 
34 #include <BaseType.h>
35 #include <D4Attributes.h>
36 #include <XMLWriter.h>
37 
38 #include <BESIndent.h>
39 #include <BESDebug.h>
40 #include <BESLog.h>
41 #include <BESInternalError.h>
42 
43 #include "DmrppRequestHandler.h"
44 #include "DmrppCommon.h"
45 #include "DmrppArray.h"
46 #include "Chunk.h"
47 #include "util.h"
48 
49 using namespace std;
50 using namespace libdap;
51 
52 #define prolog std::string("DmrppCommon::").append(__func__).append("() - ")
53 
54 namespace dmrpp {
55 
56 // Used with BESDEBUG
57 static const string dmrpp_3 = "dmrpp:3";
58 static const string dmrpp_4 = "dmrpp:4";
59 
60 bool DmrppCommon::d_print_chunks = false;
61 string DmrppCommon::d_dmrpp_ns = "http://xml.opendap.org/dap/dmrpp/1.0.0#";
62 string DmrppCommon::d_ns_prefix = "dmrpp";
63 
74 void join_threads(pthread_t threads[], unsigned int num_threads)
75 {
76  int status;
77  for (unsigned int i = 0; i < num_threads; ++i) {
78  if (threads[i]) {
79  BESDEBUG(dmrpp_3, "Join thread " << i << " after an exception was caught." << endl);
80  string *error = NULL;
81  if ((status = pthread_join(threads[i], (void **) &error)) < 0) {
82  BESDEBUG(dmrpp_3, "Could not join thread " << i << ", " << strerror(status)<< endl);
83  // ERROR_LOG("Failed to join thread " << i << " during clean up from an exception: " << strerror(status) << endl);
84  }
85  else if (error != NULL) {
86  BESDEBUG(dmrpp_3, "Joined thread " << i << ", error exit: " << *error << endl);
87  // ERROR_LOG("Joined thread " << i << ", error exit" << *error << endl);
88  }
89  else {
90  BESDEBUG(dmrpp_3, "Joined thread " << i << ", successful exit." << endl);
91  }
92  }
93  }
94 }
95 
105 void DmrppCommon::parse_chunk_dimension_sizes(const string &chunk_dims_string)
106 {
107  d_chunk_dimension_sizes.clear();
108 
109  if (chunk_dims_string.empty()) return;
110 
111  string chunk_dims = chunk_dims_string;
112  // If the input is anything other than integers and spaces, throw
113  if (chunk_dims.find_first_not_of("1234567890 ") != string::npos)
114  throw BESInternalError("while processing chunk dimension information, illegal character(s)", __FILE__, __LINE__);
115 
116  // istringstream can parse this kind of input more easily. jhrg 4/10/18
117 
118  string space(" ");
119  size_t strPos = 0;
120  string strVal;
121 
122  // Are there spaces or multiple values?
123  if (chunk_dims.find(space) != string::npos) {
124  // Process space delimited content
125  while ((strPos = chunk_dims.find(space)) != string::npos) {
126  strVal = chunk_dims.substr(0, strPos);
127 
128  d_chunk_dimension_sizes.push_back(strtol(strVal.c_str(), NULL, 10));
129  chunk_dims.erase(0, strPos + space.length());
130  }
131  }
132 
133  // If it's multi valued there's still one more value left to process
134  // If it's single valued the same is true, so let's ingest that.
135  d_chunk_dimension_sizes.push_back(strtol(chunk_dims.c_str(), NULL, 10));
136 }
137 
144 void DmrppCommon::ingest_compression_type(const string &compression_type_string)
145 {
146  if (compression_type_string.empty()) return;
147 
148  // Clear previous state
149  d_deflate = false;
150  d_shuffle = false;
151 
152  string deflate("deflate");
153  string shuffle("shuffle");
154 
155  // Process content
156  if (compression_type_string.find(deflate) != string::npos) {
157  d_deflate = true;
158  }
159 
160  if (compression_type_string.find(shuffle) != string::npos) {
161  d_shuffle = true;
162  }
163 }
164 
170  void DmrppCommon::ingest_byte_order(const string &byte_order_string) {
171 
172  if (byte_order_string.empty()) return;
173 
174  // Process content
175  if (byte_order_string.compare("LE") == 0) {
176  d_byte_order = "LE";
177  d_twiddle_bytes = is_host_big_endian();
178  } else {
179  if (byte_order_string.compare("BE") == 0) {
180  d_byte_order = "BE";
181  d_twiddle_bytes = !(is_host_big_endian());
182  } else {
183  throw BESInternalError("Did not recognize byteOrder.", __FILE__, __LINE__);
184  }
185  }
186  }
187 
188 #if 0
189 std::string DmrppCommon::get_byte_order()
190  {
191  return d_byte_order;
192  }
193 #endif
194 
199 unsigned long DmrppCommon::add_chunk(
200  const string &data_url,
201  const string &byte_order,
202  unsigned long long size,
203  unsigned long long offset,
204  const string &position_in_array)
205 
206 {
207  vector<unsigned int> cpia_vector;
208  Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
209  return add_chunk(data_url, byte_order, size, offset, cpia_vector);
210 }
211 
212 unsigned long DmrppCommon::add_chunk(
213  const string &data_url,
214  const string &byte_order,
215  unsigned long long size,
216  unsigned long long offset,
217  const vector<unsigned int> &position_in_array)
218 {
219  std::shared_ptr<Chunk> chunk(new Chunk(data_url, byte_order, size, offset, position_in_array));
220 #if 0
221  auto array = dynamic_cast<dmrpp::DmrppArray *>(this);
222  if(!array){
223  stringstream msg;
224  msg << prolog << "ERROR DmrrpCommon::add_chunk() may only be called on an instance of DmrppArray. ";
225  msg << "The variable";
226  auto bt = dynamic_cast<libdap::BaseType *>(this);
227  if(bt){
228  msg << " " << bt->type_name() << " " << bt->name();
229  }
230  msg << " is not an instance of DmrppArray.";
231  msg << "this: " << (void **) this << " ";
232  msg << "byte_order: " << byte_order << " ";
233  msg << "size: " << size << " ";
234  msg << "offset: " << offset << " ";
235  throw BESInternalError(msg.str(),__FILE__, __LINE__);
236  }
237 
238  if(d_super_chunks.empty())
239  d_super_chunks.push_back( shared_ptr<SuperChunk>(new SuperChunk()));
240 
241  auto currentSuperChunk = d_super_chunks.back();
242 
243  bool chunk_was_added = currentSuperChunk->add_chunk(chunk);
244  if(!chunk_was_added){
245  if(currentSuperChunk->empty()){
246  stringstream msg;
247  msg << prolog << "ERROR! Failed to add a Chunk to an empty SuperChunk. This should not happen.";
248  throw BESInternalError(msg.str(),__FILE__,__LINE__);
249  }
250  // This means that the chunk was not contiguous with the currentSuperChunk
251  currentSuperChunk = shared_ptr<SuperChunk>(new SuperChunk());
252  chunk_was_added = currentSuperChunk->add_chunk(chunk);
253  if(!chunk_was_added) {
254  stringstream msg;
255  msg << prolog << "ERROR! Failed to add a Chunk to an empty SuperChunk. This should not happen.";
256  throw BESInternalError(msg.str(),__FILE__,__LINE__);
257  }
258  d_super_chunks.push_back(currentSuperChunk);
259  }
260 #endif
261 
262  d_chunks.push_back(chunk);
263  return d_chunks.size();
264 }
265 
283 char *
284 DmrppCommon::read_atomic(const string &name)
285 {
286  auto chunk_refs = get_chunks();
287 
288  if (chunk_refs.size() != 1)
289  throw BESInternalError(string("Expected only a single chunk for variable ") + name, __FILE__, __LINE__);
290 
291  auto chunk = chunk_refs[0];
292 
293  chunk->read_chunk();
294 
295  return chunk->get_rbuf();
296 }
297 
301 void
302 DmrppCommon::print_chunks_element(XMLWriter &xml, const string &name_space)
303 {
304  // Start element "chunks" with dmrpp namespace and attributes:
305  if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunks", NULL) < 0)
306  throw BESInternalError("Could not start chunks element.", __FILE__, __LINE__);
307 
308  string compression = "";
309  if (is_shuffle_compression() && is_deflate_compression())
310  compression = "deflate shuffle";
311  else if (is_shuffle_compression())
312  compression.append("shuffle");
313  else if (is_deflate_compression())
314  compression.append("deflate");
315 
316  if (!compression.empty())
317  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "compressionType", (const xmlChar*) compression.c_str()) < 0)
318  throw BESInternalError("Could not write compression attribute.", __FILE__, __LINE__);
319 
320 
321  if(!get_chunks().empty()){
322  auto first_chunk = get_chunks().front();
323  if (!first_chunk->get_byte_order().empty()) {
324  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar *) "byteOrder",
325  (const xmlChar *) first_chunk->get_byte_order().c_str()) < 0)
326  throw BESInternalError("Could not write attribute byteOrder", __FILE__, __LINE__);
327  }
328  }
329 
330  if (d_chunk_dimension_sizes.size() > 0) {
331  // Write element "chunkDimensionSizes" with dmrpp namespace:
332  ostringstream oss;
333  copy(d_chunk_dimension_sizes.begin(), d_chunk_dimension_sizes.end(), ostream_iterator<unsigned int>(oss, " "));
334  string sizes = oss.str();
335  sizes.erase(sizes.size() - 1, 1); // trim the trailing space
336 
337  if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar*) name_space.c_str(), (const xmlChar*) "chunkDimensionSizes", NULL,
338  (const xmlChar*) sizes.c_str()) < 0) throw BESInternalError("Could not write chunkDimensionSizes attribute.", __FILE__, __LINE__);
339  }
340 
341  // Start elements "chunk" with dmrpp namespace and attributes:
342  // for (vector<Chunk>::iterator i = get_chunks().begin(), e = get_chunks().end(); i != e; ++i) {
343 
344  for(auto chunk: get_chunks()){
345 
346  if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunk", NULL) < 0)
347  throw BESInternalError("Could not start element chunk", __FILE__, __LINE__);
348 
349  // Get offset string:
350  ostringstream offset;
351  offset << chunk->get_offset();
352  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "offset", (const xmlChar*) offset.str().c_str()) < 0)
353  throw BESInternalError("Could not write attribute offset", __FILE__, __LINE__);
354 
355  // Get nBytes string:
356  ostringstream nBytes;
357  nBytes << chunk->get_size();
358  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "nBytes", (const xmlChar*) nBytes.str().c_str()) < 0)
359  throw BESInternalError("Could not write attribute nBytes", __FILE__, __LINE__);
360 
361  if (chunk->get_position_in_array().size() > 0) {
362  // Get position in array string:
363  vector<unsigned int> pia = chunk->get_position_in_array();
364  ostringstream oss;
365  oss << "[";
366  copy(pia.begin(), pia.end(), ostream_iterator<unsigned int>(oss, ","));
367  string pia_str = oss.str();
368  if (pia.size() > 0) pia_str.replace(pia_str.size() - 1, 1, "]"); // replace the trailing ',' with ']'
369  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "chunkPositionInArray", (const xmlChar*) pia_str.c_str()) < 0)
370  throw BESInternalError("Could not write attribute position in array", __FILE__, __LINE__);
371  }
372 
373  // End element "chunk":
374  if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunk element", __FILE__, __LINE__);
375  }
376 
377  if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunks element", __FILE__, __LINE__);
378 }
379 
383 void
384 DmrppCommon::print_compact_element(XMLWriter &xml, const string &name_space, const std::string &encoded)
385 {
386  // Write element "compact" with dmrpp namespace:
387  ostringstream oss;
388  copy(encoded.begin(), encoded.end(), ostream_iterator<char>(oss, ""));
389  string sizes = oss.str();
390 
391  if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar *) name_space.c_str(),
392  (const xmlChar *) "compact", NULL,
393  (const xmlChar *) sizes.c_str()) < 0)
394  throw BESInternalError("Could not write compact element.", __FILE__, __LINE__);
395 }
396 
407 void DmrppCommon::print_dmrpp(XMLWriter &xml, bool constrained /*false*/)
408 {
409  BaseType &bt = dynamic_cast<BaseType&>(*this);
410  if (constrained && !bt.send_p())
411  return;
412 
413  if (xmlTextWriterStartElement(xml.get_writer(), (const xmlChar*)bt.type_name().c_str()) < 0)
414  throw InternalErr(__FILE__, __LINE__, "Could not write " + bt.type_name() + " element");
415 
416  if (!bt.name().empty())
417  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "name", (const xmlChar*)bt.name().c_str()) < 0)
418  throw InternalErr(__FILE__, __LINE__, "Could not write attribute for name");
419 
420  if (bt.is_dap4())
421  bt.attributes()->print_dap4(xml);
422 
423  if (!bt.is_dap4() && bt.get_attr_table().get_size() > 0)
424  bt.get_attr_table().print_xml_writer(xml);
425 
426  // This is the code added to libdap::BaseType::print_dap4(). jhrg 5/10/18
427  if (DmrppCommon::d_print_chunks && get_immutable_chunks().size() > 0)
428  print_chunks_element(xml, DmrppCommon::d_ns_prefix);
429 
430  if (xmlTextWriterEndElement(xml.get_writer()) < 0)
431  throw InternalErr(__FILE__, __LINE__, "Could not end " + bt.type_name() + " element");
432 }
433 
434 void DmrppCommon::dump(ostream & strm) const
435 {
436  strm << BESIndent::LMarg << "is_deflate: " << (is_deflate_compression() ? "true" : "false") << endl;
437  strm << BESIndent::LMarg << "is_shuffle_compression: " << (is_shuffle_compression() ? "true" : "false") << endl;
438 
439  const vector<unsigned int> &chunk_dim_sizes = get_chunk_dimension_sizes();
440 
441  strm << BESIndent::LMarg << "chunk dimension sizes: [";
442  for (unsigned int i = 0; i < chunk_dim_sizes.size(); i++) {
443  strm << (i ? "][" : "") << chunk_dim_sizes[i];
444  }
445  strm << "]" << endl;
446 
447  auto chunk_refs = get_immutable_chunks();
448  strm << BESIndent::LMarg << "Chunks (aka chunks):" << (chunk_refs.size() ? "" : "None Found.") << endl;
449  BESIndent::Indent();
450  for (auto & chunk_ref : chunk_refs) {
451  strm << BESIndent::LMarg;
452  chunk_ref->dump(strm);
453  strm << endl;
454  }
455 }
456 
457 } // namepsace dmrpp
458 
exception thrown if internal error encountered
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:64