32 #ifdef HAVE_TR1_FUNCTIONAL 33 #include <tr1/functional> 41 #include <DapXmlNamespaces.h> 42 #include <ConstraintEvaluator.h> 43 #include <DDXParserSAX2.h> 47 #include <D4EnumDefs.h> 48 #include <D4Dimensions.h> 51 #include <D4ParserSax2.h> 58 #ifdef DAP2_STORED_RESULTS 59 #include <XDRStreamMarshaller.h> 60 #include <XDRStreamUnMarshaller.h> 63 #include <chunked_istream.h> 64 #include <D4StreamUnMarshaller.h> 67 #include <mime_util.h> 70 #include "BESStoredDapResultCache.h" 71 #include "BESDapResponseBuilder.h" 72 #include "BESInternalError.h" 75 #include "TheBESKeys.h" 78 #ifdef HAVE_TR1_FUNCTIONAL 79 #define HASH_OBJ std::tr1::hash 81 #define HASH_OBJ std::hash 85 #define BES_DATA_ROOT "BES.Data.RootDirectory" 86 #define BES_CATALOG_ROOT "BES.Catalog.catalog.RootDirectory" 92 const string BESStoredDapResultCache::SUBDIR_KEY =
"DAP.StoredResultsCache.subdir";
93 const string BESStoredDapResultCache::PREFIX_KEY =
"DAP.StoredResultsCache.prefix";
94 const string BESStoredDapResultCache::SIZE_KEY =
"DAP.StoredResultsCache.size";
96 unsigned long BESStoredDapResultCache::getCacheSizeFromConfig()
100 unsigned long size_in_megabytes = 0;
103 istringstream iss(size);
104 iss >> size_in_megabytes;
107 string msg =
"[ERROR] BESStoredDapResultCache::getCacheSize() - The BES Key " + SIZE_KEY
108 +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
109 BESDEBUG(
"cache", msg << endl);
112 return size_in_megabytes;
115 string BESStoredDapResultCache::getSubDirFromConfig()
122 string msg =
"[ERROR] BESStoredDapResultCache::getSubDirFromConfig() - The BES Key " + SUBDIR_KEY
123 +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
124 BESDEBUG(
"cache", msg << endl);
128 while (*subdir.begin() ==
'/' && subdir.length() > 0) {
129 subdir = subdir.substr(1);
138 string BESStoredDapResultCache::getResultPrefixFromConfig()
147 string msg =
"[ERROR] BESStoredDapResultCache::getResultPrefix() - The BES Key " + PREFIX_KEY
148 +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
149 BESDEBUG(
"cache", msg << endl);
156 string BESStoredDapResultCache::getBesDataRootDirFromConfig()
159 string cacheDir =
"";
164 string msg = ((string)
"[ERROR] BESStoredDapResultCache::getStoredResultsDir() - Neither the BES Key ")
165 + BES_CATALOG_ROOT +
"or the BES key " + BES_DATA_ROOT
166 +
" have been set! One MUST be set to utilize the Stored Result Caching system. ";
167 BESDEBUG(
"cache", msg << endl);
175 BESStoredDapResultCache::BESStoredDapResultCache()
177 BESDEBUG(
"cache",
"BESStoredDapResultCache::BESStoredDapResultCache() - BEGIN" << endl);
179 d_storedResultsSubdir = getSubDirFromConfig();
180 d_dataRootDir = getBesDataRootDirFromConfig();
183 d_resultFilePrefix = getResultPrefixFromConfig();
184 d_maxCacheSize = getCacheSizeFromConfig();
187 "BESStoredDapResultCache() - Stored results cache configuration params: " << resultsDir <<
", " << d_resultFilePrefix <<
", " << d_maxCacheSize << endl);
189 initialize(resultsDir, d_resultFilePrefix, d_maxCacheSize);
191 BESDEBUG(
"cache",
"BESStoredDapResultCache::BESStoredDapResultCache() - END" << endl);
197 BESStoredDapResultCache::BESStoredDapResultCache(
const string &data_root_dir,
const string &stored_results_subdir,
198 const string &result_file_prefix,
unsigned long long max_cache_size)
201 d_storedResultsSubdir = stored_results_subdir;
202 d_dataRootDir = data_root_dir;
203 d_resultFilePrefix = result_file_prefix;
204 d_maxCacheSize = max_cache_size;
205 initialize(
BESUtil::assemblePath(d_dataRootDir, stored_results_subdir), d_resultFilePrefix, d_maxCacheSize);
210 const string &result_file_prefix,
unsigned long long max_cache_size)
212 if (d_instance == 0) {
213 if (dir_exists(data_root_dir)) {
218 atexit(delete_instance);
223 "[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
236 if (d_instance == 0) {
240 atexit(delete_instance);
245 "[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
261 bool BESStoredDapResultCache::is_valid(
const string &cache_file_name,
const string &dataset)
266 off_t entry_size = 0;
267 time_t entry_time = 0;
269 if (stat(cache_file_name.c_str(), &buf) == 0) {
270 entry_size = buf.st_size;
271 entry_time = buf.st_mtime;
277 if (entry_size == 0)
return false;
279 time_t dataset_time = entry_time;
280 if (stat(dataset.c_str(), &buf) == 0) {
281 dataset_time = buf.st_mtime;
289 if (dataset_time > entry_time)
return false;
294 #ifdef DAP2_STORED_RESULTS 306 bool BESStoredDapResultCache::read_dap2_data_from_cache(
const string &cache_file_name, DDS *fdds)
309 "BESStoredDapResultCache::read_dap2_data_from_cache() - Opening cache file: " << cache_file_name << endl);
314 if (get_read_lock(cache_file_name, fd)) {
316 ifstream data(cache_file_name.c_str());
319 string mime = get_next_mime_header(data);
320 while (!mime.empty()) {
321 mime = get_next_mime_header(data);
325 DDXParser ddx_parser(fdds->get_factory());
328 string boundary = read_multipart_boundary(data);
330 "BESStoredDapResultCache::read_dap2_data_from_cache() - MPM Boundary: " << boundary << endl);
332 read_multipart_headers(data,
"text/xml", dods_ddx);
335 "BESStoredDapResultCache::read_dap2_data_from_cache() - Read the multipart haeaders" << endl);
341 ddx_parser.intern_stream(data, fdds, data_cid, boundary);
343 "BESStoredDapResultCache::read_dap2_data_from_cache() - Dataset name: " << fdds->get_dataset_name() << endl);
347 "BESStoredDapResultCache::read_dap2_data_from_cache() - DDX Parser Error: " << e.get_error_message() << endl);
353 "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (before): " << data_cid << endl);
354 data_cid = cid_to_header_value(data_cid);
356 "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (after): " << data_cid << endl);
360 read_multipart_headers(data,
"application/octet-stream", dods_data_ddx, data_cid);
365 XDRStreamUnMarshaller um(data);
366 for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
367 (*i)->deserialize(um, fdds);
371 unlock_and_close(cache_file_name );
375 BESDEBUG(
"cache",
"BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
382 "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
384 if (fd != -1) unlock_and_close(cache_file_name );
401 bool BESStoredDapResultCache::read_dap4_data_from_cache(
const string &cache_file_name, libdap::DMR *dmr)
403 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - BEGIN" << endl);
408 if (get_read_lock(cache_file_name, fd)) {
410 "BESStoredDapResultCache::read_dap4_data_from_cache() - Opening cache file: " << cache_file_name << endl);
411 fstream in(cache_file_name.c_str(), ios::in | ios::binary);
422 chunked_istream cis(in, CHUNK_SIZE);
429 int chunk_size = cis.read_next_chunk();
432 "BESStoredDapResultCache::read_dap4_data_from_cache() - First chunk_size: " << chunk_size << endl);
434 if (chunk_size == EOF) {
435 throw InternalErr(__FILE__, __LINE__,
436 "BESStoredDapResultCache::read_dap4_data_from_cache() - Failed to read first chunk from file. Chunk size = EOF (aka " 437 + libdap::long_to_string(EOF) +
")");
441 char chunk[chunk_size];
442 cis.read(chunk, chunk_size);
443 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - Read first chunk." << endl);
448 parser.intern(chunk, chunk_size - 2, dmr, debug);
449 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - Parsed first chunk." << endl);
451 D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
453 dmr->root()->deserialize(um, *dmr);
454 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - Deserialized data." << endl);
456 BESDEBUG(
"cache",
"BESStoredDapResultCache::read_dap4_data_from_cache() - END" << endl);
459 unlock_and_close(cache_file_name );
465 BESDEBUG(
"cache",
"BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
473 "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
475 if (fd != -1) unlock_and_close(cache_file_name );
480 #ifdef DAP2_STORED_RESULTS 486 BESStoredDapResultCache::get_cached_dap2_data_ddx(
const string &cache_file_name, BaseTypeFactory *factory,
487 const string &filename)
490 "BESStoredDapResultCache::get_cached_dap2_data_ddx() - Reading cache for " << cache_file_name << endl);
492 DDS *fdds =
new DDS(factory);
494 if (read_dap2_data_from_cache(cache_file_name, fdds)) {
496 fdds->filename(filename);
499 BESDEBUG(
"cache",
"DDS Filename: " << fdds->filename() << endl);
500 BESDEBUG(
"cache",
"DDS Dataset name: " << fdds->get_dataset_name() << endl);
502 fdds->set_factory(0);
506 DDS::Vars_iter i = fdds->var_begin();
507 while (i != fdds->var_end()) {
508 (*i)->set_read_p(
true);
509 (*i++)->set_send_p(
true);
528 const string &filename)
531 "BESStoredDapResultCache::get_cached_dap4_data() - Reading cache for " << cache_file_name << endl);
533 DMR *fdmr =
new DMR(factory);
535 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cached_dap4_data() - DMR Filename: " << fdmr->filename() << endl);
536 fdmr->set_filename(filename);
538 if (read_dap4_data_from_cache(cache_file_name, fdmr)) {
540 "BESStoredDapResultCache::get_cached_dap4_data() - DMR Dataset name: " << fdmr->name() << endl);
542 fdmr->set_factory(0);
546 fdmr->root()->set_send_p(
true);
547 fdmr->root()->set_read_p(
true);
555 #ifdef DAP2_STORED_RESULTS 560 string BESStoredDapResultCache::store_dap2_result(DDS &dds,
const string &constraint,
BESDapResponseBuilder *rb,
561 ConstraintEvaluator *eval)
563 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - BEGIN" << endl);
565 BaseTypeFactory factory;
569 string local_id = get_stored_result_local_id(dds.filename(), constraint, DAP_3_2);
570 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - local_id: "<< local_id << endl);
571 string cache_file_name = get_cache_file_name(local_id,
false);
572 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - cache_file_name: "<< cache_file_name << endl);
579 if (!is_valid(cache_file_name, dds.filename())) purge_file(cache_file_name);
581 if (get_read_lock(cache_file_name, fd)) {
583 "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
585 else if (create_and_lock(cache_file_name, fd)) {
589 "BESStoredDapResultCache::store_dap2_result() - cache_file_name " << cache_file_name <<
", constraint: " << constraint << endl);
591 #if 0 // I shut this off because we know that the constraint and functions have already been evaluated - ndp 595 eval->parse_constraint(constraint, *fdds);
597 if (eval->function_clauses()) {
598 DDS *temp_fdds = eval->eval_function_clauses(*fdds);
604 ofstream data_stream(cache_file_name.c_str());
606 throw InternalErr(__FILE__, __LINE__,
607 "Could not open '" + cache_file_name +
"' to write cached response.");
609 string start =
"dataddx_cache_start", boundary =
"dataddx_cache_boundary";
613 ConstraintEvaluator eval;
617 dds.set_dap_version(
"3.2");
624 set_mime_multipart(data_stream, boundary, start, dods_data_ddx, x_plain,
627 rb->serialize_dap2_data_ddx(data_stream, (DDS**) &dds, eval, boundary, start);
630 data_stream << CRLF <<
"--" << boundary <<
"--" << CRLF;
637 exclusive_to_shared_lock(fd);
642 unsigned long long size = update_cache_info(cache_file_name);
643 if (cache_too_big(size)) update_and_purge(cache_file_name);
648 else if (get_read_lock(cache_file_name, fd)) {
650 "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
653 throw InternalErr(__FILE__, __LINE__,
654 "BESStoredDapResultCache::store_dap2_result() - Cache error during function invocation.");
658 "BESStoredDapResultCache::store_dap2_result() - unlocking and closing cache file "<< cache_file_name << endl);
659 unlock_and_close(cache_file_name);
663 "BESStoredDapResultCache::store_dap2_result() - caught exception, unlocking cache and re-throw." << endl);
669 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap2_result() - END (local_id=`"<< local_id <<
"')" << endl);
681 string BESStoredDapResultCache::get_stored_result_local_id(
const string &dataset,
const string &ce,
682 libdap::DAPVersion version)
684 BESDEBUG(
"cache",
"get_stored_result_local_id() - BEGIN. dataset: " << dataset <<
", ce: " << ce << endl);
685 std::ostringstream ostr;
686 HASH_OBJ<std::string> str_hash;
687 string name = dataset +
"#" + ce;
688 ostr << str_hash(name);
689 string hashed_name = ostr.str();
690 BESDEBUG(
"cache",
"get_stored_result_local_id() - hashed_name: " << hashed_name << endl);
694 #ifdef DAP2_STORED_RESULTS 700 suffix =
".data_ddx";
708 throw BESInternalError(
"BESStoredDapResultCache::get_stored_result_local_id() - Unrecognized DAP version!!",
713 BESDEBUG(
"cache",
"get_stored_result_local_id() - Data file suffix: " << suffix << endl);
715 string local_id = d_resultFilePrefix + hashed_name + suffix;
716 BESDEBUG(
"cache",
"get_stored_result_local_id() - file: " << local_id << endl);
720 BESDEBUG(
"cache",
"get_stored_result_local_id() - END. local_id: " << local_id << endl);
730 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - BEGIN" << endl);
732 BaseTypeFactory factory;
736 string local_id = get_stored_result_local_id(dmr.filename(), constraint, DAP_4_0);
737 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - local_id: "<< local_id << endl);
738 string cache_file_name = get_cache_file_name(local_id,
false);
739 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - cache_file_name: "<< cache_file_name << endl);
746 if (!is_valid(cache_file_name, dmr.filename())) {
748 "BESStoredDapResultCache::store_dap4_result() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
749 purge_file(cache_file_name);
752 if (get_read_lock(cache_file_name, fd)) {
754 "BESStoredDapResultCache::store_dap4_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
756 else if (create_and_lock(cache_file_name, fd)) {
760 "BESStoredDapResultCache::store_dap4_result() - cache_file_name: " << cache_file_name <<
", constraint: " << constraint << endl);
762 ofstream data_stream(cache_file_name.c_str());
764 throw InternalErr(__FILE__, __LINE__,
765 "Could not open '" + cache_file_name +
"' to write cached response.");
776 exclusive_to_shared_lock(fd);
781 unsigned long long size = update_cache_info(cache_file_name);
782 if (cache_too_big(size)) update_and_purge(cache_file_name);
786 else if (get_read_lock(cache_file_name, fd)) {
788 "BESStoredDapResultCache::store_dap4_result() - Couldn't create and lock file, But I got a read lock. " "Result may have been created by another process. " "Not rewriting file: " << cache_file_name << endl);
791 throw InternalErr(__FILE__, __LINE__,
792 "BESStoredDapResultCache::store_dap4_result() - Cache error during function invocation.");
796 "BESStoredDapResultCache::store_dap4_result() - unlocking and closing cache file "<< cache_file_name << endl);
797 unlock_and_close(cache_file_name);
801 "BESStoredDapResultCache::store_dap4_result() - caught exception, unlocking cache and re-throw." << endl);
807 BESDEBUG(
"cache",
"BESStoredDapResultCache::store_dap4_result() - END (local_id=`"<< local_id <<
"')" << endl);
exception thrown if inernal error encountered
static string lowercase(const string &s)
virtual std::string get_message()
get the error message for this exception
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
libdap::DMR * get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory, const string &filename)
virtual std::string get_dataset_name() const
Get the dataset name.
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
static BESStoredDapResultCache * get_instance()
static bool IsSet(const string &flagName)
see if the debug context flagName is set to true
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
static BESKeys * TheKeys()
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)