32 #ifdef HAVE_TR1_FUNCTIONAL
33 #include <tr1/functional>
40 #include <ConstraintEvaluator.h>
41 #include <DDXParserSAX2.h>
42 #include <XDRStreamMarshaller.h>
43 #include <XDRStreamUnMarshaller.h>
46 #include <mime_util.h>
58 #ifdef HAVE_TR1_FUNCTIONAL
59 #define HASH_OBJ std::tr1::hash
61 #define HASH_OBJ std::hash
66 #define BES_DATA_ROOT "BES.Data.RootDirectory"
67 #define BES_CATALOG_ROOT "BES.Catalog.catalog.RootDirectory"
71 using namespace libdap;
83 unsigned long size_in_megabytes = 0;
86 istringstream iss(size);
87 iss >> size_in_megabytes;
90 string msg =
"[ERROR] BESStoreResultCache::getCacheSize() - The BES Key " + SIZE_KEY +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
94 return size_in_megabytes;
103 string msg =
"[ERROR] BESStoreResultCache::getDefaultSubDir() - The BES Key " + SUBDIR_KEY +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
108 while(*subdir.begin() ==
'/' && subdir.length()>0){
109 subdir = subdir.substr(1);
127 string msg =
"[ERROR] BESStoreResultCache::getResultPrefix() - The BES Key " + PREFIX_KEY +
" is not set! It MUST be set to utilize the Stored Result Caching system. ";
136 BESDEBUG(
"cache",
"BESStoreResultCache::getDefaultCacheDir() - BEGIN" << endl);
139 string cacheDir =
"";
144 string msg = ((string)
"[ERROR] BESStoreResultCache::getStoredResultsDir() - Neither the BES Key ") +
BES_CATALOG_ROOT +
145 "or the BES key " +
BES_DATA_ROOT +
" have been set! One MUST be set to utilize the Stored Result Caching system. ";
150 BESDEBUG(
"cache",
"BESStoreResultCache::getDefaultCacheDir() - Using data directory: " << cacheDir << endl);
153 if(*cacheDir.rbegin() !=
'/')
156 string subDir = getSubDirFromConfig();
159 BESDEBUG(
"cache",
"BESStoreResultCache::getDefaultCacheDir() - Stored Results Directory: " << cacheDir << endl);
161 BESDEBUG(
"cache",
"BESStoreResultCache::getDefaultCacheDir() - END" << endl);
166 BESStoredDapResultCache::BESStoredDapResultCache(){
167 BESDEBUG(
"cache",
"BESStoreResultCache::BESStoreResultCache() - BEGIN" << endl);
169 string resultsDir = getStoredResultsDirFromConfig();
170 string resultPrefix = getResultPrefixFromConfig();
171 unsigned long size_in_megabytes = getCacheSizeFromConfig();
173 BESDEBUG(
"cache",
"BESStoreResultCache() - Cache config params: " << resultsDir <<
", " << resultPrefix <<
", " << size_in_megabytes << endl);
178 if (!resultsDir.empty() && size_in_megabytes > 0)
179 initialize(resultsDir, resultPrefix, size_in_megabytes);
181 BESDEBUG(
"cache",
"BESStoreResultCache::BESStoreResultCache() - END" << endl);
198 BESStoredDapResultCache::BESStoredDapResultCache(
const string &stored_results_dir,
const string &prefix,
unsigned long long size):
BESFileLockingCache(stored_results_dir,prefix,size) {
217 if (d_instance == 0){
218 if(dir_exists(cache_dir)){
223 BESDEBUG(
"cache",
"BESStoreResultCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
236 if (d_instance == 0) {
242 BESDEBUG(
"cache",
"BESStoreResultCache::get_instance(): Failed to obtain cache! msg: " << bie.
get_message() << endl);
252 void BESStoredDapResultCache::delete_instance() {
253 BESDEBUG(
"cache",
"BESStoreResultCache::delete_instance() - Deleting singleton BESStoreResultCache instance." << endl);
269 bool BESStoredDapResultCache::is_valid(
const string &cache_file_name,
const string &dataset)
274 off_t entry_size = 0;
275 time_t entry_time = 0;
277 if (stat(cache_file_name.c_str(), &buf) == 0) {
278 entry_size = buf.st_size;
279 entry_time = buf.st_mtime;
288 time_t dataset_time = entry_time;
289 if (stat(dataset.c_str(), &buf) == 0) {
290 dataset_time = buf.st_mtime;
298 if (dataset_time > entry_time)
315 void BESStoredDapResultCache::read_data_from_cache(
const string &cache_file_name, DDS *fdds)
317 BESDEBUG(
"cache",
"Opening cache file: " << cache_file_name << endl);
318 ifstream data(cache_file_name.c_str());
321 string mime = get_next_mime_header(data);
322 while (!mime.empty()) {
323 mime = get_next_mime_header(data);
327 DDXParser ddx_parser(fdds->get_factory());
330 string boundary = read_multipart_boundary(data);
331 BESDEBUG(
"cache",
"MPM Boundary: " << boundary << endl);
333 read_multipart_headers(data,
"text/xml", dap4_ddx);
335 BESDEBUG(
"cache",
"Read the multipart haeaders" << endl);
341 ddx_parser.intern_stream(data, fdds, data_cid, boundary);
342 BESDEBUG(
"cache",
"Dataset name: " << fdds->get_dataset_name() << endl);
345 BESDEBUG(
"cache",
"DDX Parser Error: " << e.get_error_message() << endl);
350 BESDEBUG(
"cache",
"Data CID (before): " << data_cid << endl);
351 data_cid = cid_to_header_value(data_cid);
352 BESDEBUG(
"cache",
"Data CID (after): " << data_cid << endl);
356 read_multipart_headers(data,
"application/octet-stream", dap4_data, data_cid);
361 XDRStreamUnMarshaller um(data);
362 for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
363 (*i)->deserialize(um, fdds);
372 BESStoredDapResultCache::get_cached_data_ddx(
const string &cache_file_name, BaseTypeFactory *factory,
const string &filename)
374 BESDEBUG(
"cache",
"Reading cache for " << cache_file_name << endl);
376 DDS *fdds =
new DDS(factory);
378 fdds->filename(filename) ;
381 read_data_from_cache(cache_file_name, fdds);
383 BESDEBUG(
"cache",
"DDS Filename: " << fdds->filename() << endl);
384 BESDEBUG(
"cache",
"DDS Dataset name: " << fdds->get_dataset_name() << endl);
386 fdds->set_factory( 0 ) ;
390 DDS::Vars_iter i = fdds->var_begin();
391 while(i != fdds->var_end()) {
392 (*i)->set_read_p(
true );
393 (*i++)->set_send_p(
true);
434 BESDEBUG(
"cache",
"BESStoredDapResultCache::cache_dataset() - BEGIN" << endl );
436 BaseTypeFactory factory;
441 string cache_file_name =
get_cache_file_name(build_stored_result_file_name(dds.filename(), constraint),
false);
448 if (!is_valid(cache_file_name, dds.filename()))
452 BESDEBUG(
"cache",
"BESStoredDapResultCache::cache_dataset() - function ce (change)- cached hit: " << cache_file_name << endl);
453 fdds = get_cached_data_ddx(cache_file_name, &factory, dds.filename());
458 BESDEBUG(
"cache",
"BESStoredDapResultCache::cache_dataset() - function ce - caching " << cache_file_name <<
", constraint: " << constraint << endl);
461 eval->parse_constraint(constraint, *fdds);
463 if (eval->function_clauses()) {
464 DDS *temp_fdds = eval->eval_function_clauses(*fdds);
469 ofstream data_stream(cache_file_name.c_str());
471 throw InternalErr(__FILE__, __LINE__,
"Could not open '" + cache_file_name +
"' to write cached response.");
473 string start=
"dataddx_cache_start", boundary=
"dataddx_cache_boundary";
477 ConstraintEvaluator eval;
481 fdds->set_dap_version(
"3.2");
488 set_mime_multipart(data_stream, boundary, start, dap4_data_ddx, x_plain, last_modified_time(rb->
get_dataset_name()));
493 data_stream <<
CRLF <<
"--" << boundary <<
"--" <<
CRLF;
512 BESDEBUG(
"cache",
"BESStoredDapResultCache::cache_dataset() - function ce - cached hit: " << cache_file_name << endl);
513 fdds = get_cached_data_ddx(cache_file_name, &factory, dds.get_dataset_name());
516 throw InternalErr(__FILE__, __LINE__,
"BESStoredDapResultCache::cache_dataset() - Cache error during function invocation.");
520 BESDEBUG(
"cache",
"BESStoredDapResultCache::cache_dataset() - caught exception, unlocking cache and re-throw." << endl );
528 cache_token = cache_file_name;
529 BESDEBUG(
"cache",
"BESStoredDapResultCache::cache_dataset() - END (cache_token=`"<< cache_token <<
"'" << endl );
541 BESStoredDapResultCache::build_stored_result_file_name(
const string &dataset,
const string &ce)
543 BESDEBUG(
"cache",
"build_stored_result_file_name() - BEGIN. dataset: " << dataset <<
", ce: " << ce << endl);
544 std::ostringstream ostr;
545 HASH_OBJ<std::string> str_hash;
546 string name = dataset +
"#" + ce;
547 ostr << str_hash(name);
548 string hashed_name = ostr.str();
550 BESDEBUG(
"cache",
"build_stored_result_file_name(): hashed_name: " << hashed_name << endl);
575 while(*target.begin() ==
'/' && target.length()>0){
576 target = target.substr(1);
579 throw BESInternalError(
"BESStoredDapResultCache: The target cache file name must not be made of only the '/' character. Srsly.", __FILE__, __LINE__);
584 if(*cacheDir.rbegin() !=
'/')
589 while(*prefix.begin() ==
'/' && prefix.length()>0){
590 prefix = prefix.substr(1);
593 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cache_file_name() - cacheDir: '" << cacheDir <<
"'" << endl);
594 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cache_file_name() - prefix: '" << prefix <<
"'" << endl);
595 BESDEBUG(
"cache",
"BESStoredDapResultCache::get_cache_file_name() - target: '" << target <<
"'" << endl);
598 BESDEBUG(
"cache",
"[WARNING] BESStoredDapResultCache::get_cache_file_name() - The parameter 'mangle' is ignored!" << endl);
602 return cacheDir + prefix + target;
virtual void unlock_cache()
Unlock the cache info file.
exception thrown if inernal error encountered
virtual libdap::DDS * cache_dataset(libdap::DDS &dds, const std::string &constraint, BESDapResponseBuilder *rb, libdap::ConstraintEvaluator *eval, std::string &cache_token)
Get the cached DDS object.
virtual bool create_and_lock(const string &target, int &fd)
Create a file in the cache and lock it for write access.
static string lowercase(const string &s)
Convert a string to all lower case.
static string getResultPrefixFromConfig()
virtual string get_cache_file_name(const string &src, bool mangle=false)
Build the name of file that will holds the uncompressed data from 'src' in the cache.
static const string SIZE_KEY
const string getCacheDirectory()
static string getStoredResultsDirFromConfig()
Implementation of a caching mechanism for compressed data.
virtual string get_message()
get the error message for this exception
static string getSubDirFromConfig()
static BESStoredDapResultCache * get_instance()
Get the default instance of the BESStoreResultCache object.
virtual void purge_file(const string &file)
Purge a single file from the cache.
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big...
static const string SUBDIR_KEY
static const string PREFIX_KEY
const string getCacheFilePrefix()
virtual bool get_read_lock(const string &target, int &fd)
Get a read-only lock on the file if it exists.
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
virtual void update_and_purge(const string &new_file)
Purge files from the cache.
static unsigned long getCacheSizeFromConfig()
virtual unsigned long long update_cache_info(const string &target)
Update the cache info file to include 'target'.
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock.
This class is used to build responses for/by the BES.
#define BESDEBUG(x, y)
macro used to send debug information to the debug stream
virtual void dataset_constraint_ddx(std::ostream &out, libdap::DDS &dds, libdap::ConstraintEvaluator &eval, const std::string &boundary, const std::string &start, bool ce_eval=true)
Build/return the DDX and the BLOB part of the DAP3.x data response.
static BESKeys * TheKeys()
virtual std::string get_dataset_name() const
The ``dataset name'' is the filename or other string that the filter program will use to access the d...
This class is used to cache DAP2 response objects.