35 #include <sys/types.h>
62 #define MKDIR(a,b) _mkdir((a))
63 #define REMOVE(a) do { \
64 int s = remove((a)); \
66 throw InternalErr(__FILE__, __LINE__, "Cache error; could not remove file: " + long_to_string(s)); \
68 #define MKSTEMP(a) _open(_mktemp((a)),_O_CREAT,_S_IREAD|_S_IWRITE)
69 #define DIR_SEPARATOR_CHAR '\\'
70 #define DIR_SEPARATOR_STR "\\"
72 #define MKDIR(a,b) mkdir((a), (b))
73 #define MKSTEMP(a) mkstemp((a))
74 #define DIR_SEPARATOR_CHAR '/'
75 #define DIR_SEPARATOR_STR "/"
78 #define CACHE_META ".meta"
79 #define CACHE_INDEX ".index"
80 #define CACHE_EMPTY_ETAG "@cache@"
82 #define NO_LM_EXPIRATION 24*3600 // 24 hours
83 #define MAX_LM_EXPIRATION 48*3600 // Max expiration from LM
88 #define LM_EXPIRATION(t) (min((MAX_LM_EXPIRATION), static_cast<int>((t) / 10)))
105 for (
const char *ptr = url.c_str(); *ptr; ptr++)
111 HTTPCacheTable::HTTPCacheTable(
const string &cache_root,
int block_size) :
112 d_cache_root(cache_root), d_block_size(block_size), d_current_size(0), d_new_entries(0)
120 d_cache_table[i] = 0;
131 DBG2(cerr <<
"Deleting CacheEntry: " << e << endl);
141 for_each(cp->begin(), cp->end(), delete_cache_entry);
144 delete get_cache_table()[i];
145 get_cache_table()[i] = 0;
149 delete[] d_cache_table;
159 class DeleteExpired :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
165 d_time(t), d_table(table) {
170 void operator()(HTTPCacheTable::CacheEntry *&e) {
171 if (e && !e->readers && (e->freshness_lifetime
172 < (e->corrected_initial_age + (d_time - e->response_time)))) {
173 DBG(cerr <<
"Deleting expired cache entry: " << e->url << endl);
186 for_each(slot->begin(), slot->end(), DeleteExpired(*
this, time));
187 slot->erase(
remove(slot->begin(), slot->end(),
199 class DeleteByHits :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
205 d_table(table), d_hits(hits) {
208 void operator()(HTTPCacheTable::CacheEntry *&e) {
209 if (e && !e->readers && e->hits <= d_hits) {
210 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
220 if (get_cache_table()[cnt]) {
222 for_each(slot->begin(), slot->end(), DeleteByHits(*
this, hits));
223 slot->erase(
remove(slot->begin(), slot->end(),
235 class DeleteBySize :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
241 d_table(table), d_size(size) {
244 void operator()(HTTPCacheTable::CacheEntry *&e) {
245 if (e && !e->readers && e->size > d_size) {
246 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
255 if (get_cache_table()[cnt]) {
257 for_each(slot->begin(), slot->end(), DeleteBySize(*
this, size));
258 slot->erase(
remove(slot->begin(), slot->end(),
298 FILE *fp = fopen(d_cache_index.c_str(),
"r");
306 while (!feof(fp) && fgets(line, 1024, fp)) {
308 DBG2(cerr << line << endl);
311 int res = fclose(fp) ;
313 DBG(cerr <<
"HTTPCache::cache_index_read - Failed to close " << (
void *)fp << endl);
333 istringstream iss(line);
335 iss >> entry->cachename;
342 iss >> entry->expires;
348 iss >> entry->freshness_lifetime;
349 iss >> entry->response_time;
350 iss >> entry->corrected_initial_age;
352 iss >> entry->must_revalidate;
359 class WriteOneCacheEntry :
360 public unary_function<HTTPCacheTable::CacheEntry *, void>
366 WriteOneCacheEntry(FILE *fp) : d_fp(fp)
369 void operator()(HTTPCacheTable::CacheEntry *e)
371 if (e && fprintf(d_fp,
372 "%s %s %s %ld %ld %ld %c %d %d %ld %ld %ld %c\r\n",
374 e->cachename.c_str(),
379 e->range ?
'1' :
'0',
382 (long)(e->freshness_lifetime),
383 (long)(e->response_time),
384 (long)(e->corrected_initial_age),
385 e->must_revalidate ?
'1' :
'0') < 0)
386 throw Error(
"Cache Index. Error writing cache index\n");
402 DBG(cerr <<
"Cache Index. Writing index " << d_cache_index << endl);
406 if ((fp = fopen(d_cache_index.c_str(),
"wb")) == NULL) {
407 throw Error(
string(
"Cache Index. Can't open `") + d_cache_index
408 +
string(
"' for writing"));
417 for_each(cp->begin(), cp->end(), WriteOneCacheEntry(fp));
421 int res = fclose(fp);
423 DBG(cerr <<
"HTTPCache::cache_index_write - Failed to close "
424 << (
void *)fp << endl);
447 struct stat stat_info;
450 path << d_cache_root << hash;
451 string p = path.str();
453 if (stat(p.c_str(), &stat_info) == -1) {
454 DBG2(cerr <<
"Cache....... Create dir " << p << endl);
455 if (
MKDIR(p.c_str(), 0777) < 0) {
456 DBG2(cerr <<
"Cache....... Can't create..." << endl);
457 throw Error(
"Could not create cache slot to hold response! Check the write permissions on your disk cache directory. Cache root: " + d_cache_root +
".");
461 DBG2(cerr <<
"Cache....... Directory " << p <<
" already exists"
487 hash_dir +=
"\\dodsXXXXXX";
489 hash_dir +=
"/dodsXXXXXX";
494 vector<char> templat(hash_dir.size() + 1);
495 strncpy(&templat[0], hash_dir.c_str(), hash_dir.size() + 1);
509 throw Error(
"The HTTP Cache could not create a file to hold the response; it will not be cached.");
512 entry->cachename = &templat[0];
520 entry_disk_space(
int size,
unsigned int block_size)
522 unsigned int num_of_blocks = (size + block_size) / block_size;
524 DBG(cerr <<
"size: " << size <<
", block_size: " << block_size
525 <<
", num_of_blocks: " << num_of_blocks << endl);
527 return num_of_blocks * block_size;
542 int hash = entry->hash;
544 throw InternalErr(__FILE__, __LINE__,
"Hash value too large!");
546 if (!d_cache_table[hash])
549 d_cache_table[hash]->push_back(entry);
551 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size
552 <<
", entry->size: " << entry->size <<
", block size: " << d_block_size
555 d_current_size += entry_disk_space(entry->size, d_block_size);
557 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size << endl);
566 HTTPCacheTable::get_locked_entry_from_cache_table(
const string &url)
568 return get_locked_entry_from_cache_table(
get_hash(url), url);
579 HTTPCacheTable::get_locked_entry_from_cache_table(
int hash,
const string &url)
581 DBG(cerr <<
"url: " << url <<
"; hash: " << hash << endl);
582 DBG(cerr <<
"d_cache_table: " << hex << d_cache_table << dec << endl);
583 if (d_cache_table[hash]) {
588 if ((*i) && (*i)->url == url) {
589 (*i)->lock_read_response();
604 HTTPCacheTable::CacheEntry *
608 if (d_cache_table[hash]) {
613 if ((*i) && (*i)->url == url) {
614 (*i)->lock_write_response();
636 throw InternalErr(__FILE__, __LINE__,
"Tried to delete a cache entry that is in use.");
638 REMOVE(entry->cachename.c_str());
643 unsigned int eds = entry_disk_space(entry->size,
get_block_size());
651 class DeleteCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void>
658 : d_url(url), d_cache_table(c)
661 void operator()(HTTPCacheTable::CacheEntry *&e)
663 if (e && e->url == d_url) {
664 e->lock_write_response();
666 e->unlock_write_response();
682 if (d_cache_table[hash]) {
684 for_each(cp->begin(), cp->end(), DeleteCacheEntry(
this, url));
692 class DeleteUnlockedCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
700 void operator()(HTTPCacheTable::CacheEntry *&e)
717 for_each(slot->begin(), slot->end(), DeleteUnlockedCacheEntry(*
this));
741 entry->response_time = time(NULL);
742 time_t apparent_age = max(0, static_cast<int>(entry->response_time - entry->date));
743 time_t corrected_received_age = max(apparent_age, entry->age);
744 time_t response_delay = entry->response_time - request_time;
745 entry->corrected_initial_age = corrected_received_age + response_delay;
750 time_t freshness_lifetime = entry->max_age;
751 if (freshness_lifetime < 0) {
752 if (entry->expires < 0) {
754 freshness_lifetime = default_expiration;
761 freshness_lifetime = entry->expires - entry->date;
764 entry->freshness_lifetime = max(0, static_cast<int>(freshness_lifetime));
766 DBG2(cerr <<
"Cache....... Received Age " << entry->age
767 <<
", corrected " << entry->corrected_initial_age
768 <<
", freshness lifetime " << entry->freshness_lifetime << endl);
783 const vector<string> &headers)
785 vector<string>::const_iterator i;
786 for (i = headers.begin(); i != headers.end(); ++i) {
791 string::size_type colon = (*i).find(
':');
794 if (colon == string::npos)
797 string header = (*i).substr(0, (*i).find(
':'));
798 string value = (*i).substr((*i).find(
": ") + 2);
799 DBG2(cerr <<
"Header: " << header << endl);
DBG2(cerr <<
"Value: " << value << endl);
801 if (header ==
"ETag") {
804 else if (header ==
"Last-Modified") {
807 else if (header ==
"Expires") {
810 else if (header ==
"Date") {
813 else if (header ==
"Age") {
816 else if (header ==
"Content-Length") {
817 unsigned long clength = strtoul(value.c_str(), 0, 0);
818 if (clength > max_entry_size)
821 else if (header ==
"Cache-Control") {
825 if (value ==
"no-cache" || value ==
"no-store")
830 else if (value ==
"must-revalidate")
831 entry->must_revalidate =
true;
832 else if (value.find(
"max-age") != string::npos) {
833 string max_age = value.substr(value.find(
"=" + 1));
845 d_locked_entries[body] = entry;
852 throw InternalErr(
"There is no cache entry for the response given.");
854 d_locked_entries.erase(body);
857 if (entry->readers < 0)
858 throw InternalErr(
"An unlocked entry was released");
862 return !d_locked_entries.empty();
void remove_cache_entry(HTTPCacheTable::CacheEntry *entry)
const int CACHE_TABLE_SIZE
time_t parse_time(const char *str, bool expand)
void create_location(CacheEntry *entry)
void unlock_read_response()
void parse_headers(HTTPCacheTable::CacheEntry *entry, unsigned long max_entry_size, const vector< string > &headers)
void increment_new_entries()
void add_entry_to_cache_table(CacheEntry *entry)
vector< CacheEntry * > CacheEntries
void calculate_time(HTTPCacheTable::CacheEntry *entry, int default_expiration, time_t request_time)
void delete_by_size(unsigned int size)
CacheEntries::iterator CacheEntriesIter
void set_no_cache(bool state)
A class for software fault reporting.
void delete_all_entries()
void set_current_size(unsigned long sz)
int get_hash(const string &url)
void bind_entry_to_data(CacheEntry *entry, FILE *body)
void delete_expired_entries(time_t time=0)
unsigned long get_current_size() const
CacheEntry * cache_index_parse_line(const char *line)
unsigned int get_block_size() const
void remove_entry_from_cache_table(const string &url)
CacheEntry * get_write_locked_entry_from_cache_table(const string &url)
void delete_by_hits(int hits)
A class for error processing.
string create_hash_directory(int hash)
bool cache_index_delete()
void uncouple_entry_from_data(FILE *body)
bool is_locked_read_responses()