From a8c0b78fac5f9b451804477c165d33e2ec52f8eb Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 4 Aug 2017 13:03:53 -0500 Subject: [PATCH 01/23] std string --- cmd/traffic_cache_tool/CacheTool.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index febd4ed323e..7b213a0fa77 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -203,7 +203,7 @@ Stripe::Stripe(Span *span, Bytes start, CacheStoreBlocks len) : _span(span), _st (uint64_t)_len.count()); printf("hash id of stripe is hash of %s\n", hash_text); ink_code_md5((unsigned char *)hash_text, strlen(hash_text), (unsigned char *)&hash_id); - hashText = hash_text; + hashText.assign(hash_text,strlen(hash_text)); } bool From e02a1875e28838b9ab32ebcbf64609f98b7b4cd3 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 4 Aug 2017 13:25:31 -0500 Subject: [PATCH 02/23] std string --- cmd/traffic_cache_tool/CacheTool.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 7b213a0fa77..5eda7a2f808 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -165,7 +165,8 @@ struct Stripe { int64_t _buckets; ///< Number of buckets per segment. int64_t _segments; ///< Number of segments. - const char *hashText = nullptr; + + std::string hashText; /// Meta copies, indexed by A/B then HEAD/FOOT. StripeMeta _meta[2][2]; From e4b19dfaaef7cd92c3f993e4faf4fcc18539920c Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 4 Aug 2017 14:46:23 -0500 Subject: [PATCH 03/23] debug msg --- iocore/cache/Cache.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index b84f4634e1f..69a51ee8059 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2492,6 +2492,8 @@ Cache::lookup(Continuation *cont, const CacheKey *key, CacheFragType type, const } Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); ProxyMutex *mutex = cont->mutex.get(); CacheVC *c = new_CacheVC(cont); SET_CONTINUATION_HANDLER(c, &CacheVC::openReadStartHead); From a660d674e1e811f058202ab1fc9dadcd7db92e89 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 4 Aug 2017 15:18:57 -0500 Subject: [PATCH 04/23] debug --- iocore/cache/Cache.cc | 2 ++ iocore/cache/CacheLink.cc | 4 +++- iocore/cache/CacheRead.cc | 4 ++++ iocore/cache/CacheWrite.cc | 4 ++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 69a51ee8059..b24988f7baa 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2613,6 +2613,8 @@ Cache::remove(Continuation *cont, const CacheKey *key, CacheFragType type, const CACHE_TRY_LOCK(lock, cont->mutex, this_ethread()); ink_assert(lock.is_locked()); Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); // coverity[var_decl] Dir result; dir_clear(&result); // initialized here, set result empty so we can recognize missed lock diff --git a/iocore/cache/CacheLink.cc b/iocore/cache/CacheLink.cc index 4b3f9f30ed3..a59ebb687d3 100644 --- a/iocore/cache/CacheLink.cc +++ b/iocore/cache/CacheLink.cc @@ -41,6 +41,8 @@ Cache::link(Continuation *cont, const CacheKey *from, const CacheKey *to, CacheF c->earliest_key = *to; c->buf = new_IOBufferData(BUFFER_SIZE_INDEX_512); + char hashStr[33]; + Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); #ifdef DEBUG Doc *doc = (Doc *)c->buf->data(); memcpy(doc->data(), to, sizeof(*to)); // doublecheck @@ -82,7 +84,7 @@ Cache::deref(Continuation *cont, const CacheKey *key, CacheFragType type, const } ink_assert(caches[type] == this); - + char hashStr[33]; Vol *vol = key_to_vol(key, hostname, host_len); Dir result; Dir *last_collision = nullptr; diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index b9a6ccd196e..1ed6b951a54 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -37,6 +37,8 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheFragType type, co ink_assert(caches[type] == this); Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; @@ -103,6 +105,8 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, ink_assert(caches[type] == this); Vol *vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index 58719803910..d344cb95de3 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -1653,6 +1653,8 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheFragType frag_ty c->vio.op = VIO::WRITE; c->base_stat = cache_write_active_stat; c->vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); Vol *vol = c->vol; CACHE_INCREMENT_DYN_STAT(c->base_stat + CACHE_STAT_ACTIVE); c->first_key = c->key = *key; @@ -1730,6 +1732,8 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheHTTPInfo *info, c->earliest_key = c->key; c->frag_type = CACHE_FRAG_TYPE_HTTP; c->vol = key_to_vol(key, hostname, host_len); + char hashStr[33]; + Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); Vol *vol = c->vol; c->info = info; if (c->info && (uintptr_t)info != CACHE_ALLOW_MULTIPLE_WRITES) { From 1a074558ad1598c043947c7218b6039cdfc739bb Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 4 Aug 2017 16:07:16 -0500 Subject: [PATCH 05/23] correct ebug tag --- iocore/cache/Cache.cc | 4 ++-- iocore/cache/CacheLink.cc | 5 +++-- iocore/cache/CacheRead.cc | 4 ++-- iocore/cache/CacheWrite.cc | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index b24988f7baa..47638303024 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2493,7 +2493,7 @@ Cache::lookup(Continuation *cont, const CacheKey *key, CacheFragType type, const Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); ProxyMutex *mutex = cont->mutex.get(); CacheVC *c = new_CacheVC(cont); SET_CONTINUATION_HANDLER(c, &CacheVC::openReadStartHead); @@ -2614,7 +2614,7 @@ Cache::remove(Continuation *cont, const CacheKey *key, CacheFragType type, const ink_assert(lock.is_locked()); Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); // coverity[var_decl] Dir result; dir_clear(&result); // initialized here, set result empty so we can recognize missed lock diff --git a/iocore/cache/CacheLink.cc b/iocore/cache/CacheLink.cc index a59ebb687d3..c95180b88db 100644 --- a/iocore/cache/CacheLink.cc +++ b/iocore/cache/CacheLink.cc @@ -42,7 +42,7 @@ Cache::link(Continuation *cont, const CacheKey *from, const CacheKey *to, CacheF c->buf = new_IOBufferData(BUFFER_SIZE_INDEX_512); char hashStr[33]; - Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); #ifdef DEBUG Doc *doc = (Doc *)c->buf->data(); memcpy(doc->data(), to, sizeof(*to)); // doublecheck @@ -84,10 +84,11 @@ Cache::deref(Continuation *cont, const CacheKey *key, CacheFragType type, const } ink_assert(caches[type] == this); - char hashStr[33]; Vol *vol = key_to_vol(key, hostname, host_len); Dir result; Dir *last_collision = nullptr; + char hashStr[33]; + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); CacheVC *c = nullptr; { MUTEX_TRY_LOCK(lock, vol->mutex, cont->mutex->thread_holding); diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index 1ed6b951a54..bf8a4041c8c 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -38,7 +38,7 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheFragType type, co Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; @@ -106,7 +106,7 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index d344cb95de3..f830afd7834 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -1654,7 +1654,7 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheFragType frag_ty c->base_stat = cache_write_active_stat; c->vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); Vol *vol = c->vol; CACHE_INCREMENT_DYN_STAT(c->base_stat + CACHE_STAT_ACTIVE); c->first_key = c->key = *key; @@ -1733,7 +1733,7 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheHTTPInfo *info, c->frag_type = CACHE_FRAG_TYPE_HTTP; c->vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("Cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); + Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); Vol *vol = c->vol; c->info = info; if (c->info && (uintptr_t)info != CACHE_ALLOW_MULTIPLE_WRITES) { From 032a70985d841e87fa438e00ce67b97561f42cb6 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 4 Aug 2017 16:20:26 -0500 Subject: [PATCH 06/23] fix debug --- cmd/traffic_cache_tool/CacheTool.cc | 2 +- iocore/cache/Cache.cc | 4 ++-- iocore/cache/CacheLink.cc | 4 ++-- iocore/cache/CacheRead.cc | 4 ++-- iocore/cache/CacheWrite.cc | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 5eda7a2f808..c525c14ad98 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -1405,7 +1405,7 @@ Find_Stripe(FilePath const &input_file_path) ink_code_md5((unsigned char *)host.data(), host.size(), (unsigned char *)&hash); Stripe *stripe_ = cache.key_to_stripe(&hash, host.data(), host.size()); printf("hash of %.*s is %s: Stripe %s \n", (int)host.size(), host.data(), - ink_code_to_hex_str(hashStr, (unsigned char *)&hash), stripe_->hashText); + ink_code_to_hex_str(hashStr, (unsigned char *)&hash), stripe_->hashText.data()); } } diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 47638303024..3c6455a0736 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2493,7 +2493,7 @@ Cache::lookup(Continuation *cont, const CacheKey *key, CacheFragType type, const Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); ProxyMutex *mutex = cont->mutex.get(); CacheVC *c = new_CacheVC(cont); SET_CONTINUATION_HANDLER(c, &CacheVC::openReadStartHead); @@ -2614,7 +2614,7 @@ Cache::remove(Continuation *cont, const CacheKey *key, CacheFragType type, const ink_assert(lock.is_locked()); Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); // coverity[var_decl] Dir result; dir_clear(&result); // initialized here, set result empty so we can recognize missed lock diff --git a/iocore/cache/CacheLink.cc b/iocore/cache/CacheLink.cc index c95180b88db..43fec1b5e89 100644 --- a/iocore/cache/CacheLink.cc +++ b/iocore/cache/CacheLink.cc @@ -42,7 +42,7 @@ Cache::link(Continuation *cont, const CacheKey *from, const CacheKey *to, CacheF c->buf = new_IOBufferData(BUFFER_SIZE_INDEX_512); char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); #ifdef DEBUG Doc *doc = (Doc *)c->buf->data(); memcpy(doc->data(), to, sizeof(*to)); // doublecheck @@ -88,7 +88,7 @@ Cache::deref(Continuation *cont, const CacheKey *key, CacheFragType type, const Dir result; Dir *last_collision = nullptr; char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); CacheVC *c = nullptr; { MUTEX_TRY_LOCK(lock, vol->mutex, cont->mutex->thread_holding); diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index bf8a4041c8c..717165666ed 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -38,7 +38,7 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheFragType type, co Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; @@ -106,7 +106,7 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index f830afd7834..87e203b8ee5 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -1654,7 +1654,7 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheFragType frag_ty c->base_stat = cache_write_active_stat; c->vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); Vol *vol = c->vol; CACHE_INCREMENT_DYN_STAT(c->base_stat + CACHE_STAT_ACTIVE); c->first_key = c->key = *key; @@ -1733,7 +1733,7 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheHTTPInfo *info, c->frag_type = CACHE_FRAG_TYPE_HTTP; c->vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %s assigned vol hashID %s : ID %s",hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); + Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); Vol *vol = c->vol; c->info = info; if (c->info && (uintptr_t)info != CACHE_ALLOW_MULTIPLE_WRITES) { From 018daf9700c308e6a953cb85ccacffb81e6fa870 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Mon, 7 Aug 2017 09:37:19 -0500 Subject: [PATCH 07/23] debug --- proxy/hdrs/URL.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 372c2f3e529..cf4727d4f63 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1821,6 +1821,7 @@ url_host_MD5_get(URLImpl *url, INK_MD5 *md5) // Especially since it's in_port_t for url_MD5_get. int port = url_canonicalize_port(url->m_url_type, url->m_port); ctx.update(&port, sizeof(port)); + Debug("cache","%s://%s:%d",url->m_ptr_scheme,url->m_ptr_host,port); ctx.finalize(*md5); } From 1915c554818f603c30a93839af6c14301889f1af Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Mon, 7 Aug 2017 10:06:28 -0500 Subject: [PATCH 08/23] debug --- proxy/hdrs/URL.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index cf4727d4f63..a5764844846 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1696,7 +1696,7 @@ url_MD5_get_fast(const URLImpl *url, CryptoContext &ctx, CryptoHash *hash, cache if (generation != -1) { ctx.update(&generation, sizeof(generation)); } - +Debug("cache","fast string to be hashed: %s%d",buffer,generation); ctx.finalize(hash); } @@ -1772,7 +1772,7 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca if (generation != -1) { ctx.update(&generation, sizeof(generation)); } - + Debug("cache","string to be hashed: %s%d%d",buffer,port,generation); ctx.finalize(hash); } From 3a0201cc1d87a4e7db9deb523571abdecd2e4a2d Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Mon, 7 Aug 2017 10:23:32 -0500 Subject: [PATCH 09/23] debug --- proxy/hdrs/URL.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index a5764844846..446063adb28 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1696,8 +1696,9 @@ url_MD5_get_fast(const URLImpl *url, CryptoContext &ctx, CryptoHash *hash, cache if (generation != -1) { ctx.update(&generation, sizeof(generation)); } -Debug("cache","fast string to be hashed: %s%d",buffer,generation); ctx.finalize(hash); + char hashStr[33]; + Debug("cache","fast string to be hashed: %s%d => %s",buffer,generation,ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); } static inline void From 2f19390c6fc0677a8eb086785c49157021ee35db Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Mon, 7 Aug 2017 10:31:18 -0500 Subject: [PATCH 10/23] debug --- proxy/hdrs/URL.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 446063adb28..f8818ee3e65 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1773,8 +1773,10 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca if (generation != -1) { ctx.update(&generation, sizeof(generation)); } - Debug("cache","string to be hashed: %s%d%d",buffer,port,generation); ctx.finalize(hash); + char hashStr[33]; + Debug("cache","string to be hashed: %s%d%d => %s",buffer,port,generation, ink_code_to_hex_str(hashStr, (unsigned char *) &hash)); + } void From dfc439599460a74a7e9e390ba3844d7a18df8e7b Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Mon, 7 Aug 2017 15:12:09 -0500 Subject: [PATCH 11/23] de --- proxy/hdrs/URL.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index f8818ee3e65..2bcf59eb6e9 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1756,6 +1756,8 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } if (p == e) { + char hashStr[33]; + Debug("cache","p==e string to be hashed: %s%d%d => %s",buffer,port,generation, ink_code_to_hex_str(hashStr, (unsigned char *) &hash)); ctx.update(buffer, BUFSIZE); p = buffer; } @@ -1764,6 +1766,8 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } if (p != buffer) { + char hashStr[33]; + Debug("cache","p!=buffer string to be hashed: %s%d%d => %s",buffer,port,generation, ink_code_to_hex_str(hashStr, (unsigned char *) &hash)); ctx.update(buffer, p - buffer); } From 6d3176de2a7be9419d4a0219f47af4a6c437c941 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Mon, 7 Aug 2017 17:04:38 -0500 Subject: [PATCH 12/23] hash matches with ats --- cmd/traffic_cache_tool/CacheTool.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index c525c14ad98..8c526f05663 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -43,6 +43,7 @@ #include #include #include +#include using ts::Bytes; using ts::Megabytes; @@ -1391,6 +1392,19 @@ Clear_Spans(int argc, char *argv[]) Errata Find_Stripe(FilePath const &input_file_path) { + //scheme=http user=u password=p host=172.28.56.109 path=somepath query=somequery port=1234 + // url format: scheme://hostname:port/somepath;params?somequery + //user, password,path,query,port are optional; scheme and host are required + MD5Context ctx; + INK_MD5 hashT; + + char hashStr[33]; + char* h= "http://:@172.28.56.109/file7;?"; + ctx.update(h,strlen(h)); + in_port_t port = 5005; + ctx.update(&port,sizeof(port)); + ctx.finalize(hashT); + printf("hash is %s: \n", ink_code_to_hex_str(hashStr, (unsigned char *)&hashT)); Errata zret; Cache cache; if (input_file_path) @@ -1401,7 +1415,6 @@ Find_Stripe(FilePath const &input_file_path) cache.build_stripe_hash_table(); for (auto host : cache.URLset) { INK_MD5 hash; - char hashStr[33]; ink_code_md5((unsigned char *)host.data(), host.size(), (unsigned char *)&hash); Stripe *stripe_ = cache.key_to_stripe(&hash, host.data(), host.size()); printf("hash of %.*s is %s: Stripe %s \n", (int)host.size(), host.data(), From f3fc604e365da9474240bf8845d770f7ccbba6a3 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Thu, 10 Aug 2017 17:10:51 -0500 Subject: [PATCH 13/23] parsing half done --- cmd/traffic_cache_tool/CacheDefs.h | 63 +++++++++++++++++- cmd/traffic_cache_tool/CacheTool.cc | 79 ++++++++++++++++------ cmd/traffic_cache_tool/Command.cc | 13 ++-- cmd/traffic_cache_tool/Command.h | 100 +++++++++++++++++----------- cmd/traffic_cache_tool/Makefile.am | 3 +- iocore/cache/Cache.cc | 8 ++- iocore/cache/CacheLink.cc | 8 ++- iocore/cache/CacheRead.cc | 6 +- iocore/cache/CacheWrite.cc | 12 ++-- proxy/hdrs/URL.cc | 18 ++--- 10 files changed, 224 insertions(+), 86 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheDefs.h b/cmd/traffic_cache_tool/CacheDefs.h index bc589da1940..688cd7e1281 100644 --- a/cmd/traffic_cache_tool/CacheDefs.h +++ b/cmd/traffic_cache_tool/CacheDefs.h @@ -25,7 +25,8 @@ #define CACHE_DEFS_H #include #include - +#include +#include namespace tag { struct bytes { @@ -186,6 +187,66 @@ class CacheDirEntry class CacheVolume { }; + +class URLparser +{ +public: + bool verifyURL(std::string &url1); + void parseURL(); + int getPort(std::string &fullURL); + +private: + // DFA regex; +}; + +class CacheURL +{ +}; } +class DFA; +// this class matches url of the format : scheme://hostname:port/path;params?query +struct url_matcher { + // R"(^https?\:\/\/^[a-z A-Z 0-9]\.[a-z A-Z 0-9 \.]+)" + url_matcher() + { + /*if (regex.compile(R"(^https?\:\/\/^[a-z A-Z 0-9][\. a-z A-Z 0-9 ]+(\:[0-9]\/)?.*))") != 0) { + std::cout<<"Check your regular expression"< #include #include -#include using ts::Bytes; using ts::Megabytes; @@ -205,7 +204,7 @@ Stripe::Stripe(Span *span, Bytes start, CacheStoreBlocks len) : _span(span), _st (uint64_t)_len.count()); printf("hash id of stripe is hash of %s\n", hash_text); ink_code_md5((unsigned char *)hash_text, strlen(hash_text), (unsigned char *)&hash_id); - hashText.assign(hash_text,strlen(hash_text)); + hashText.assign(hash_text, strlen(hash_text)); } bool @@ -823,7 +822,7 @@ Errata Cache::loadURLs(FilePath const &path) { static const ts::StringView TAG_VOL("url"); - + ts::URLparser parser; Errata zret; ts::BulkFile cfile(path); @@ -838,6 +837,8 @@ Cache::loadURLs(FilePath const &path) } else if (0 == strcasecmp(tag, TAG_VOL)) { std::string url; url.assign(blob.begin(), blob.size()); + int port = parser.getPort(url); + std::cout << "port # " << port << std::endl; URLset.insert(url); } } @@ -1392,19 +1393,25 @@ Clear_Spans(int argc, char *argv[]) Errata Find_Stripe(FilePath const &input_file_path) { - //scheme=http user=u password=p host=172.28.56.109 path=somepath query=somequery port=1234 - // url format: scheme://hostname:port/somepath;params?somequery - //user, password,path,query,port are optional; scheme and host are required - MD5Context ctx; - INK_MD5 hashT; - - char hashStr[33]; - char* h= "http://:@172.28.56.109/file7;?"; - ctx.update(h,strlen(h)); - in_port_t port = 5005; - ctx.update(&port,sizeof(port)); - ctx.finalize(hashT); - printf("hash is %s: \n", ink_code_to_hex_str(hashStr, (unsigned char *)&hashT)); + // scheme=http user=u password=p host=172.28.56.109 path=somepath query=somequery port=1234 + // input file format: scheme://hostname:port/somepath;params?somequery user=USER password=PASS + // user, password, are optional; scheme and host are required + MD5Context ctx; + INK_MD5 hashT; + + char hashStr[33]; + char *h = "https://ss.ss.@@s.ss:900/jhfjhgsfgnn"; + // char* h= "http://:@172.28.56.109/file7;?"port <== this is the format we need + url_matcher matcher; + if (matcher.match(h)) + std::cout << h << " : is valid" << std::endl; + else + std::cout << h << " : is NOT valid" << std::endl; + ctx.update(h, strlen(h)); + in_port_t port = 5005; + ctx.update(&port, sizeof(port)); + ctx.finalize(hashT); + printf("hash is %s: \n", ink_code_to_hex_str(hashStr, (unsigned char *)&hashT)); Errata zret; Cache cache; if (input_file_path) @@ -1425,6 +1432,42 @@ Find_Stripe(FilePath const &input_file_path) return zret; } +bool +ts::URLparser::verifyURL(std::string &url1) +{ +} + +int +ts::URLparser::getPort(std::string &fullURL) +{ + url_matcher matcher; + static const ts::StringView HTTP("http"); + static const ts::StringView HTTPS("https"); + ts::StringView url(fullURL.data(), (int)fullURL.size()); + // check for scheme + ts::StringView scheme = url.splitPrefix(':'); + if ((strcasecmp(scheme, HTTP) == 0) || (strcasecmp(scheme, HTTPS) == 0)) { + url += 2; + ts::StringView hostPort = url.splitPrefix(':'); + if (hostPort) // i.e. port is present + { + ts::StringView port = url.splitPrefix('/'); + if (!port) // i.e. backslash is not present, then the rest of url must be just port + port = url; + if (matcher.portmatch(port.begin(), port.size())) { + ts::StringView text; + auto n = ts::svtoi(port, &text); + if (text == port) + return n; + } + } + return -1; + } else { + std::cout << "No scheme provided for: " << scheme.begin() << std::endl; + return -1; + } +} + int main(int argc, char *argv[]) { @@ -1454,9 +1497,7 @@ main(int argc, char *argv[]) } } - Commands - .add("list", "List elements of the cache", - []() { return List_Stripes(Cache::SpanDumpDepth::SPAN); }) + Commands.add("list", "List elements of the cache", []() { return List_Stripes(Cache::SpanDumpDepth::SPAN); }) .subCommand(std::string("stripes"), std::string("List the stripes"), []() { return List_Stripes(Cache::SpanDumpDepth::STRIPE); }); Commands.add(std::string("clear"), std::string("Clear spans"), &Clear_Spans); diff --git a/cmd/traffic_cache_tool/Command.cc b/cmd/traffic_cache_tool/Command.cc index c7a98d1310f..91912b9f9b9 100644 --- a/cmd/traffic_cache_tool/Command.cc +++ b/cmd/traffic_cache_tool/Command.cc @@ -42,25 +42,26 @@ ERR_COMMAND_TAG_NOT_FOUND(char const *tag) return ts::Errata(s.str()); } -CommandTable::Command::Command() {} +CommandTable::Command::Command() +{ +} CommandTable::Command::Command(std::string const &name, std::string const &help) : _name(name), _help(help) { } -CommandTable::Command::Command(std::string const &name, std::string const &help, LeafAction const &f) - : _name(name), _help(help) +CommandTable::Command::Command(std::string const &name, std::string const &help, LeafAction const &f) : _name(name), _help(help) { _action = f; } -CommandTable::Command::Command(std::string const &name, std::string const &help, NullaryAction const &f) - : _name(name), _help(help) +CommandTable::Command::Command(std::string const &name, std::string const &help, NullaryAction const &f) : _name(name), _help(help) { _action = f; } -CommandTable::Command::~Command() { +CommandTable::Command::~Command() +{ } CommandTable::Command & diff --git a/cmd/traffic_cache_tool/Command.h b/cmd/traffic_cache_tool/Command.h index 582f46766e7..c454721c7fd 100644 --- a/cmd/traffic_cache_tool/Command.h +++ b/cmd/traffic_cache_tool/Command.h @@ -48,7 +48,7 @@ class CommandTable /// Signature for a leaf command. using LeafAction = std::function; /// Signature for a argumentless command. - using NullaryAction = std::function; + using NullaryAction = std::function; CommandTable(); @@ -59,7 +59,7 @@ class CommandTable { typedef Command self; ///< Self reference type. public: - Command(Command && that) = default; + Command(Command &&that) = default; ~Command(); /** Add a subcommand to this command. @@ -90,60 +90,84 @@ class CommandTable /** Class to hold varying types of functions. - @internal A bit ugly, I need to do better wrapping and type erasure. + @internal A bit ugly, I need to do better wrapping and type erasure. */ - class Action { + class Action + { public: /// Type of the function stored. enum Type { - NIL, ///< Nothing / empty - LEAF, ///< Leaf action (arguments) - NULLARY, ///< Nullary action. + NIL, ///< Nothing / empty + LEAF, ///< Leaf action (arguments) + NULLARY, ///< Nullary action. }; - Action() { } - Action(Action && that) { - _type = that._type; - memcpy(_data, that._data, sizeof(_data)); - that._type = NIL; + Action() {} + Action(Action &&that) + { + _type = that._type; + memcpy(_data, that._data, sizeof(_data)); + that._type = NIL; } ~Action() { this->clear(); } - Action& operator = (LeafAction const& a) { - this->clear(); - _type = LEAF; - new (_data) LeafAction(a); - return *this; + Action & + operator=(LeafAction const &a) + { + this->clear(); + _type = LEAF; + new (_data) LeafAction(a); + return *this; } - Action& operator = (NullaryAction const& a) { - this->clear(); - _type = NULLARY; - new (_data) NullaryAction(a); - return *this; + Action & + operator=(NullaryAction const &a) + { + this->clear(); + _type = NULLARY; + new (_data) NullaryAction(a); + return *this; } - Errata invoke(int argc, char *argv[]) { - assert(LEAF == _type); - return (*reinterpret_cast(_data))(argc, argv); + Errata + invoke(int argc, char *argv[]) + { + assert(LEAF == _type); + return (*reinterpret_cast(_data))(argc, argv); } - Errata invoke() { - assert(NULLARY == _type); - return (*reinterpret_cast(_data))(); + Errata + invoke() + { + assert(NULLARY == _type); + return (*reinterpret_cast(_data))(); } - bool is_leaf() const { return LEAF == _type; } - bool is_nullary() const { return NULLARY == _type; } + bool + is_leaf() const + { + return LEAF == _type; + } + bool + is_nullary() const + { + return NULLARY == _type; + } protected: - - void clear() { - switch (_type) { - case NIL: break; - case LEAF: reinterpret_cast(_data)->~LeafAction(); break; - case NULLARY: reinterpret_cast(_data)->~NullaryAction(); break; - } - _type = NIL; + void + clear() + { + switch (_type) { + case NIL: + break; + case LEAF: + reinterpret_cast(_data)->~LeafAction(); + break; + case NULLARY: + reinterpret_cast(_data)->~NullaryAction(); + break; + } + _type = NIL; } Type _type = NIL; ///< Type of function stored. diff --git a/cmd/traffic_cache_tool/Makefile.am b/cmd/traffic_cache_tool/Makefile.am index fdb20b489b1..129e43c7584 100644 --- a/cmd/traffic_cache_tool/Makefile.am +++ b/cmd/traffic_cache_tool/Makefile.am @@ -31,6 +31,7 @@ traffic_cache_tool_LDADD = \ $(top_builddir)/lib/ts/.libs/ink_assert.o \ $(top_builddir)/lib/ts/.libs/ink_code.o \ $(top_builddir)/lib/tsconfig/.libs/Errata.o \ - @OPENSSL_LIBS@ + $(top_builddir)/lib/ts/.libs/Regex.o \ + @OPENSSL_LIBS@ @LIBPCRE@ all-am: Makefile $(PROGRAMS) diff --git a/iocore/cache/Cache.cc b/iocore/cache/Cache.cc index 3c6455a0736..e284ed17bb3 100644 --- a/iocore/cache/Cache.cc +++ b/iocore/cache/Cache.cc @@ -2491,9 +2491,10 @@ Cache::lookup(Continuation *cont, const CacheKey *key, CacheFragType type, const return ACTION_RESULT_DONE; } - Vol *vol = key_to_vol(key, hostname, host_len); + Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); ProxyMutex *mutex = cont->mutex.get(); CacheVC *c = new_CacheVC(cont); SET_CONTINUATION_HANDLER(c, &CacheVC::openReadStartHead); @@ -2614,7 +2615,8 @@ Cache::remove(Continuation *cont, const CacheKey *key, CacheFragType type, const ink_assert(lock.is_locked()); Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); // coverity[var_decl] Dir result; dir_clear(&result); // initialized here, set result empty so we can recognize missed lock diff --git a/iocore/cache/CacheLink.cc b/iocore/cache/CacheLink.cc index 43fec1b5e89..9f44d07ee80 100644 --- a/iocore/cache/CacheLink.cc +++ b/iocore/cache/CacheLink.cc @@ -42,7 +42,8 @@ Cache::link(Continuation *cont, const CacheKey *from, const CacheKey *to, CacheF c->buf = new_IOBufferData(BUFFER_SIZE_INDEX_512); char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id), c->vol->hash_text.get()); #ifdef DEBUG Doc *doc = (Doc *)c->buf->data(); memcpy(doc->data(), to, sizeof(*to)); // doublecheck @@ -88,8 +89,9 @@ Cache::deref(Continuation *cont, const CacheKey *key, CacheFragType type, const Dir result; Dir *last_collision = nullptr; char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); - CacheVC *c = nullptr; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); + CacheVC *c = nullptr; { MUTEX_TRY_LOCK(lock, vol->mutex, cont->mutex->thread_holding); if (lock.is_locked()) { diff --git a/iocore/cache/CacheRead.cc b/iocore/cache/CacheRead.cc index 717165666ed..3bd585a5ac9 100644 --- a/iocore/cache/CacheRead.cc +++ b/iocore/cache/CacheRead.cc @@ -38,7 +38,8 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheFragType type, co Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; @@ -106,7 +107,8 @@ Cache::open_read(Continuation *cont, const CacheKey *key, CacheHTTPHdr *request, Vol *vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id),vol->hash_text.get()); + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&vol->hash_id), vol->hash_text.get()); Dir result, *last_collision = nullptr; ProxyMutex *mutex = cont->mutex.get(); OpenDirEntry *od = nullptr; diff --git a/iocore/cache/CacheWrite.cc b/iocore/cache/CacheWrite.cc index 87e203b8ee5..7b8f9b8e47d 100644 --- a/iocore/cache/CacheWrite.cc +++ b/iocore/cache/CacheWrite.cc @@ -1654,8 +1654,9 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheFragType frag_ty c->base_stat = cache_write_active_stat; c->vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); - Vol *vol = c->vol; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id), c->vol->hash_text.get()); + Vol *vol = c->vol; CACHE_INCREMENT_DYN_STAT(c->base_stat + CACHE_STAT_ACTIVE); c->first_key = c->key = *key; c->frag_type = frag_type; @@ -1733,9 +1734,10 @@ Cache::open_write(Continuation *cont, const CacheKey *key, CacheHTTPInfo *info, c->frag_type = CACHE_FRAG_TYPE_HTTP; c->vol = key_to_vol(key, hostname, host_len); char hashStr[33]; - Debug("cache","Url: hostname %.*s assigned vol hashID %s : ID %s",host_len,hostname,ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id),c->vol->hash_text.get()); - Vol *vol = c->vol; - c->info = info; + Debug("cache", "Url: hostname %.*s assigned vol hashID %s : ID %s", host_len, hostname, + ink_code_to_hex_str(hashStr, (unsigned char *)&c->vol->hash_id), c->vol->hash_text.get()); + Vol *vol = c->vol; + c->info = info; if (c->info && (uintptr_t)info != CACHE_ALLOW_MULTIPLE_WRITES) { /* Update has the following code paths : diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 2bcf59eb6e9..8d10430f46d 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1698,7 +1698,7 @@ url_MD5_get_fast(const URLImpl *url, CryptoContext &ctx, CryptoHash *hash, cache } ctx.finalize(hash); char hashStr[33]; - Debug("cache","fast string to be hashed: %s%d => %s",buffer,generation,ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); + Debug("cache", "fast string to be hashed: %s%d => %s", buffer, generation, ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); } static inline void @@ -1756,8 +1756,9 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } if (p == e) { - char hashStr[33]; - Debug("cache","p==e string to be hashed: %s%d%d => %s",buffer,port,generation, ink_code_to_hex_str(hashStr, (unsigned char *) &hash)); + char hashStr[33]; + Debug("cache", "p==e string to be hashed: %s%d%d => %s", buffer, port, generation, + ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); ctx.update(buffer, BUFSIZE); p = buffer; } @@ -1766,8 +1767,9 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } if (p != buffer) { - char hashStr[33]; - Debug("cache","p!=buffer string to be hashed: %s%d%d => %s",buffer,port,generation, ink_code_to_hex_str(hashStr, (unsigned char *) &hash)); + char hashStr[33]; + Debug("cache", "p!=buffer string to be hashed: %s%d%d => %s", buffer, port, generation, + ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); ctx.update(buffer, p - buffer); } @@ -1779,8 +1781,8 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } ctx.finalize(hash); char hashStr[33]; - Debug("cache","string to be hashed: %s%d%d => %s",buffer,port,generation, ink_code_to_hex_str(hashStr, (unsigned char *) &hash)); - + Debug("cache", "string to be hashed: %s%d%d => %s", buffer, port, generation, + ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); } void @@ -1828,7 +1830,7 @@ url_host_MD5_get(URLImpl *url, INK_MD5 *md5) // Especially since it's in_port_t for url_MD5_get. int port = url_canonicalize_port(url->m_url_type, url->m_port); ctx.update(&port, sizeof(port)); - Debug("cache","%s://%s:%d",url->m_ptr_scheme,url->m_ptr_host,port); + Debug("cache", "%s://%s:%d", url->m_ptr_scheme, url->m_ptr_host, port); ctx.finalize(*md5); } From c343c6f38ae74eb66b86eb4cd68493e31e6e0946 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 11 Aug 2017 11:27:48 -0500 Subject: [PATCH 14/23] CacheURL class introduced --- cmd/traffic_cache_tool/CacheDefs.cc | 135 +++++++++ cmd/traffic_cache_tool/CacheDefs.h | 99 ++++--- cmd/traffic_cache_tool/CacheTool.cc | 444 ++++++++++++++++++++++++---- cmd/traffic_cache_tool/Makefile.am | 2 +- iocore/cache/CacheDir.cc | 2 +- proxy/hdrs/URL.cc | 6 +- 6 files changed, 581 insertions(+), 107 deletions(-) create mode 100644 cmd/traffic_cache_tool/CacheDefs.cc diff --git a/cmd/traffic_cache_tool/CacheDefs.cc b/cmd/traffic_cache_tool/CacheDefs.cc new file mode 100644 index 00000000000..f924a2d5da0 --- /dev/null +++ b/cmd/traffic_cache_tool/CacheDefs.cc @@ -0,0 +1,135 @@ +/** @file + + Main program file for Cache Tool. + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "CacheDefs.h" +#include +using namespace std; +using namespace ts; + +using ts::Errata; +namespace ts +{ +std::ostream & +operator<<(std::ostream &s, Bytes const &n) +{ + return s << n.count() << " bytes"; +} +std::ostream & +operator<<(std::ostream &s, Kilobytes const &n) +{ + return s << n.count() << " KB"; +} +std::ostream & +operator<<(std::ostream &s, Megabytes const &n) +{ + return s << n.count() << " MB"; +} +std::ostream & +operator<<(std::ostream &s, Gigabytes const &n) +{ + return s << n.count() << " GB"; +} +std::ostream & +operator<<(std::ostream &s, Terabytes const &n) +{ + return s << n.count() << " TB"; +} + +std::ostream & +operator<<(std::ostream &s, CacheStripeBlocks const &n) +{ + return s << n.count() << " stripe blocks"; +} +std::ostream & +operator<<(std::ostream &s, CacheStoreBlocks const &n) +{ + return s << n.count() << " store blocks"; +} +std::ostream & +operator<<(std::ostream &s, CacheDataBlocks const &n) +{ + return s << n.count() << " data blocks"; +} + +Errata +URLparser::parseURL(StringView URI) +{ + Errata zret; + static const StringView HTTP("http"); + static const StringView HTTPS("https"); + StringView scheme = URI.splitPrefix(':'); + if ((strcasecmp(scheme, HTTP) == 0) || (strcasecmp(scheme, HTTPS) == 0)) { + StringView hostname = URI.splitPrefix(':'); + if (!hostname) // i.e. port not present + { + } + } + + return zret; +} + +int +URLparser::getPort(std::string &fullURL, int &port_ptr, int &port_len) +{ + url_matcher matcher; + int n_port = -1; + int u_pos = -1; + + if (fullURL.find("https") == 0) { + u_pos = 8; + n_port = 443; + } else if (fullURL.find("http") == 0) { + u_pos = 7; + n_port = 80; + } + if (u_pos != -1) { + fullURL.insert(u_pos, ":@"); + static const StringView HTTP("http"); + static const StringView HTTPS("https"); + StringView url(fullURL.data(), (int)fullURL.size()); + + url += 9; + + StringView hostPort = url.splitPrefix(':'); + if (hostPort) // i.e. port is present + { + StringView port = url.splitPrefix('/'); + if (!port) // i.e. backslash is not present, then the rest of the url must be just port + port = url; + if (matcher.portmatch(port.begin(), port.size())) { + StringView text; + n_port = svtoi(port, &text); + if (text == port) { + port_ptr = fullURL.find(':', 9); + port_len = port.size(); + return n_port; + } + } + } + return n_port; + } else { + std::cout << "No scheme provided for: " << fullURL << std::endl; + return -1; + } +} +} \ No newline at end of file diff --git a/cmd/traffic_cache_tool/CacheDefs.h b/cmd/traffic_cache_tool/CacheDefs.h index 688cd7e1281..6ebffb54438 100644 --- a/cmd/traffic_cache_tool/CacheDefs.h +++ b/cmd/traffic_cache_tool/CacheDefs.h @@ -21,12 +21,15 @@ limitations under the License. */ -#if !defined(CACHE_DEFS_H) +#ifndef CACHE_DEFS_H #define CACHE_DEFS_H #include #include #include #include +#include +#include "tsconfig/Errata.h" +#include namespace tag { struct bytes { @@ -50,32 +53,6 @@ typedef Scalar<1024 * Kilobytes::SCALE, off_t, tag::bytes> Megabytes; typedef Scalar<1024 * Megabytes::SCALE, off_t, tag::bytes> Gigabytes; typedef Scalar<1024 * Gigabytes::SCALE, off_t, tag::bytes> Terabytes; -std::ostream & -operator<<(std::ostream &s, Bytes const &n) -{ - return s << n.count() << " bytes"; -} -std::ostream & -operator<<(std::ostream &s, Kilobytes const &n) -{ - return s << n.count() << " KB"; -} -std::ostream & -operator<<(std::ostream &s, Megabytes const &n) -{ - return s << n.count() << " MB"; -} -std::ostream & -operator<<(std::ostream &s, Gigabytes const &n) -{ - return s << n.count() << " GB"; -} -std::ostream & -operator<<(std::ostream &s, Terabytes const &n) -{ - return s << n.count() << " TB"; -} - // Units of allocation for stripes. typedef Scalar<128 * Megabytes::SCALE, int64_t, tag::bytes> CacheStripeBlocks; // Size measurement of cache storage. @@ -84,22 +61,6 @@ typedef Scalar<8 * Kilobytes::SCALE, int64_t, tag::bytes> CacheStoreBlocks; // Size unit for content stored in cache. typedef Scalar<512, int64_t, tag::bytes> CacheDataBlocks; -std::ostream & -operator<<(std::ostream &s, CacheStripeBlocks const &n) -{ - return s << n.count() << " stripe blocks"; -} -std::ostream & -operator<<(std::ostream &s, CacheStoreBlocks const &n) -{ - return s << n.count() << " store blocks"; -} -std::ostream & -operator<<(std::ostream &s, CacheDataBlocks const &n) -{ - return s << n.count() << " data blocks"; -} - /** A cache span is a representation of raw storage. It corresponds to a raw disk, disk partition, file, or directory. */ @@ -149,6 +110,7 @@ struct SpanHeader { @internal nee VolHeadFooter */ +// the counterpart of this structure in ATS is called VolHeaderFooter class StripeMeta { public: @@ -168,10 +130,18 @@ class StripeMeta uint32_t dirty; uint32_t sector_size; uint32_t unused; // pad out to 8 byte boundary + uint16_t freelist[1]; }; +/* + @internal struct Dir in P_CacheDir.h + * size: 10bytes + */ + class CacheDirEntry { +public: +#if 0 unsigned int offset : 24; unsigned int big : 2; unsigned int size : 6; @@ -182,6 +152,9 @@ class CacheDirEntry unsigned int token : 1; unsigned int next : 16; uint16_t offset_high; +#else + uint16_t w[5]; +#endif }; class CacheVolume @@ -192,8 +165,8 @@ class URLparser { public: bool verifyURL(std::string &url1); - void parseURL(); - int getPort(std::string &fullURL); + Errata parseURL(StringView URI); + int getPort(std::string &fullURL, int &port_ptr, int &port_len); private: // DFA regex; @@ -201,6 +174,40 @@ class URLparser class CacheURL { +public: + in_port_t port; + std::string scheme; + std::string url; + std::string hostname; + std::string path; + std::string query; + std::string params; + std::string fragments; + std::string user; + std::string password; + CacheURL(int port_, ts::StringView b_hostname, ts::StringView b_path, ts::StringView b_params, ts::StringView b_query, + ts::StringView b_fragments) + { + hostname.assign(b_hostname.begin(), b_hostname.size()); + port = port_; + path.assign(b_path.begin(), b_path.size()); + params.assign(b_params.begin(), b_params.size()); + query.assign(b_query.begin(), b_query.size()); + fragments.assign(b_fragments.begin(), b_fragments.size()); + } + + CacheURL(ts::StringView blob, int port_) + { + url.assign(blob.begin(), blob.size()); + port = port_; + } + + void + setCredential(char *p_user, int user_len, char *p_pass, int pass_len) + { + user.assign(p_user, user_len); + password.assign(p_pass, pass_len); + } }; } @@ -214,7 +221,7 @@ struct url_matcher { std::cout<<"Check your regular expression"< #include #include -#include #include #include #include @@ -59,6 +58,13 @@ constexpr int VOL_HASH_TABLE_SIZE = 32707; CacheStoreBlocks Vol_hash_alloc_size(1024); constexpr unsigned short VOL_HASH_EMPTY = 65535; constexpr int DIR_TAG_WIDTH = 12; +constexpr int DIR_DEPTH = 4; +constexpr int SIZEOF_DIR = 10; +constexpr int MAX_ENTRIES_PER_SEGMENT = (1 << 16); +constexpr int DIR_SIZE_WIDTH = 6; +constexpr int DIR_BLOCK_SIZES = 4; +constexpr int CACHE_BLOCK_SHIFT = 9; +constexpr int CACHE_BLOCK_SIZE = (1 << CACHE_BLOCK_SHIFT); // 512, smallest sector size const Bytes ts::CacheSpan::OFFSET{CacheStoreBlocks{1}}; enum { SILENT = 0, NORMAL, VERBOSE } Verbosity = NORMAL; @@ -150,6 +156,7 @@ struct Stripe { /// Load metadata for this stripe. Errata loadMeta(); + void dir_check(); /// Initialize the live data from the loaded serialized data. void updateLiveData(enum Copy c); @@ -162,6 +169,7 @@ struct Stripe { uint8_t _vol_idx = 0; ///< Volume index. uint8_t _type = 0; ///< Stripe type. int8_t _idx = -1; ///< Stripe index in span. + int agg_buf_pos = 0; int64_t _buckets; ///< Number of buckets per segment. int64_t _segments; ///< Number of segments. @@ -174,6 +182,7 @@ struct Stripe { CacheStoreBlocks _meta_pos[2][2]; /// Directory. Chunk _directory; + CacheDirEntry const *dir = nullptr; }; Stripe::Chunk::~Chunk() @@ -265,6 +274,328 @@ Stripe::updateLiveData(enum Copy c) _directory._skip = header_len; } +/* INK_ALIGN() is only to be used to align on a power of 2 boundary */ +#define INK_ALIGN(size, boundary) (((size) + ((boundary)-1)) & ~((boundary)-1)) + +#define ROUND_TO_STORE_BLOCK(_x) INK_ALIGN((_x), 8192) + +TS_INLINE int +vol_headerlen(Stripe *d) +{ + return ROUND_TO_STORE_BLOCK(sizeof(CacheDirEntry) + sizeof(uint16_t) * (d->_segments - 1)); +} + +size_t +vol_dirlen(Stripe *d) +{ + return vol_headerlen(d) + ROUND_TO_STORE_BLOCK(((size_t)d->_buckets) * DIR_DEPTH * d->_segments * SIZEOF_DIR) + + ROUND_TO_STORE_BLOCK(sizeof(CacheDirEntry)); +} + +#define dir_big(_e) ((uint32_t)((((_e)->w[1]) >> 8) & 0x3)) +#define dir_bit(_e, _w, _b) ((uint32_t)(((_e)->w[_w] >> (_b)) & 1)) +#define dir_size(_e) ((uint32_t)(((_e)->w[1]) >> 10)) +#define dir_approx_size(_e) ((dir_size(_e) + 1) * DIR_BLOCK_SIZE(dir_big(_e))) +#define dir_head(_e) dir_bit(_e, 2, 13) +#define dir_valid(_d, _e) (_d->_meta[0][0].phase == dir_phase(_e) ? vol_in_phase_valid(_d, _e) : vol_out_of_phase_valid(_d, _e)) +#define dir_tag(_e) ((uint32_t)((_e)->w[2] & ((1 << DIR_TAG_WIDTH) - 1))) +#define dir_offset(_e) \ + ((int64_t)(((uint64_t)(_e)->w[0]) | (((uint64_t)((_e)->w[1] & 0xFF)) << 16) | (((uint64_t)(_e)->w[4]) << 24))) +#define dir_set_offset(_e, _o) \ + do { \ + (_e)->w[0] = (uint16_t)_o; \ + (_e)->w[1] = (uint16_t)((((_o) >> 16) & 0xFF) | ((_e)->w[1] & 0xFF00)); \ + (_e)->w[4] = (uint16_t)((_o) >> 24); \ + } while (0) +#define dir_segment(_s, _d) vol_dir_segment(_d, _s) +#define dir_in_seg(_s, _i) ((CacheDirEntry *)(((char *)(_s)) + (SIZEOF_DIR * (_i)))) +#define dir_next(_e) (_e)->w[3] +#define dir_phase(_e) dir_bit(_e, 2, 12) +#define DIR_BLOCK_SHIFT(_i) (3 * (_i)) +#define DIR_BLOCK_SIZE(_i) (CACHE_BLOCK_SIZE << DIR_BLOCK_SHIFT(_i)) +#define dir_set_prev(_e, _o) (_e)->w[2] = (uint16_t)(_o) +#define dir_set_next(_e, _o) (_e)->w[3] = (uint16_t)(_o) + +TS_INLINE CacheDirEntry * +vol_dir_segment(Stripe *d, int s) +{ + return (CacheDirEntry *)((d->dir + d->_directory._skip) + (s * d->_buckets) * DIR_DEPTH * SIZEOF_DIR); +} + +TS_INLINE CacheDirEntry * +dir_bucket(int64_t b, CacheDirEntry *seg) +{ + return dir_in_seg(seg, b * DIR_DEPTH); +} + +TS_INLINE CacheDirEntry * +dir_from_offset(int64_t i, CacheDirEntry *seg) +{ +#if DIR_DEPTH < 5 + if (!i) + return 0; + return dir_in_seg(seg, i); +#else + i = i + ((i - 1) / (DIR_DEPTH - 1)); + return dir_in_seg(seg, i); +#endif +} + +TS_INLINE int +vol_in_phase_valid(Stripe *d, CacheDirEntry *e) +{ + return (dir_offset(e) - 1 < ((d->_meta[0][0].write_pos + d->agg_buf_pos - d->_start) / CACHE_BLOCK_SIZE)); +} + +TS_INLINE int +vol_out_of_phase_valid(Stripe *d, CacheDirEntry *e) +{ + return (dir_offset(e) - 1 >= ((d->_meta[0][0].agg_pos - d->_start) / CACHE_BLOCK_SIZE)); +} + +TS_INLINE CacheDirEntry * +next_dir(CacheDirEntry *d, CacheDirEntry *seg) +{ + int i = dir_next(d); + return dir_from_offset(i, seg); +} +#define dir_offset(_e) \ + ((int64_t)(((uint64_t)(_e)->w[0]) | (((uint64_t)((_e)->w[1] & 0xFF)) << 16) | (((uint64_t)(_e)->w[4]) << 24))) + +TS_INLINE CacheDirEntry * +dir_bucket_row(CacheDirEntry *b, int64_t i) +{ + return dir_in_seg(b, i); +} + +TS_INLINE int64_t +dir_to_offset(const CacheDirEntry *d, const CacheDirEntry *seg) +{ +#if DIR_DEPTH < 5 + return (((char *)d) - ((char *)seg)) / SIZEOF_DIR; +#else + int64_t i = (int64_t)((((char *)d) - ((char *)seg)) / SIZEOF_DIR); + i = i - (i / DIR_DEPTH); + return i; +#endif +} + +void +dir_free_entry(CacheDirEntry *e, int s, Stripe *d) +{ + CacheDirEntry *seg = dir_segment(s, d); + unsigned int fo = d->_meta[0][0].freelist[s]; + unsigned int eo = dir_to_offset(e, seg); + dir_set_next(e, fo); + if (fo) { + dir_set_prev(dir_from_offset(fo, seg), eo); + } + d->_meta[0][0].freelist[s] = eo; +} + +// adds all the directory entries +// in a segment to the segment freelist +void +dir_init_segment(int s, Stripe *d) +{ + d->_meta[0][0].freelist[s] = 0; + CacheDirEntry *seg = dir_segment(s, d); + int l, b; + memset(seg, 0, SIZEOF_DIR * DIR_DEPTH * d->_buckets); + for (l = 1; l < DIR_DEPTH; l++) { + for (b = 0; b < d->_buckets; b++) { + CacheDirEntry *bucket = dir_bucket(b, seg); + dir_free_entry(dir_bucket_row(bucket, l), s, d); + } + } +} + +// +// Cache Directory +// + +// return value 1 means no loop +// zero indicates loop +int +dir_bucket_loop_check(CacheDirEntry *start_dir, CacheDirEntry *seg) +{ + if (start_dir == nullptr) { + return 1; + } + + CacheDirEntry *p1 = start_dir; + CacheDirEntry *p2 = start_dir; + + while (p2) { + // p1 moves by one entry per iteration + assert(p1); + p1 = next_dir(p1, seg); + // p2 moves by two entries per iteration + p2 = next_dir(p2, seg); + if (p2) { + p2 = next_dir(p2, seg); + } else { + return 1; + } + + if (p2 == p1) { + return 0; // we have a loop + } + } + return 1; +} + +// break the infinite loop in directory entries +// Note : abuse of the token bit in dir entries + +int +dir_bucket_loop_fix(CacheDirEntry *start_dir, int s, Stripe *d) +{ + if (!dir_bucket_loop_check(start_dir, dir_segment(s, d))) { + dir_init_segment(s, d); + return 1; + } + return 0; +} + +int +dir_freelist_length(Stripe *d, int s) +{ + int free = 0; + CacheDirEntry *seg = dir_segment(s, d); + //TODO: check freelist[s]; ATS passes s to freelist, I don't know how that works -_- + CacheDirEntry *e = dir_from_offset(d->_meta[0][0].freelist[0], seg); + if (dir_bucket_loop_fix(e, s, d)) { + return (DIR_DEPTH - 1) * d->_buckets; + } + while (e) { + free++; + e = next_dir(e, seg); + } + return free; +} + +int +compare_ushort(void const *a, void const *b) +{ + return *static_cast(a) - *static_cast(b); +} + +void +Stripe::dir_check() +{ + static int const SEGMENT_HISTOGRAM_WIDTH = 16; + int hist[SEGMENT_HISTOGRAM_WIDTH + 1] = {0}; + unsigned short chain_tag[MAX_ENTRIES_PER_SEGMENT]; + int32_t chain_mark[MAX_ENTRIES_PER_SEGMENT]; + + this->loadMeta(); + // create raw_dir; + char *raw_dir = (char *)ats_memalign(ats_pagesize(), vol_dirlen(this)-vol_headerlen(this)-ROUND_TO_STORE_BLOCK(sizeof(StripeMeta))); + dir = (CacheDirEntry *)raw_dir; + uint64_t total_buckets = _segments * _buckets; + uint64_t total_entries = total_buckets * DIR_DEPTH; + int frag_demographics[1 << DIR_SIZE_WIDTH][DIR_BLOCK_SIZES]; + int j; + int stale = 0, in_use = 0, empty = 0; + int free = 0, head = 0, buckets_in_use = 0; + + int max_chain_length = 0; + int64_t bytes_in_use = 0; + std::cout << "Stripe '[" << hashText << "]'" << std::endl; + std::cout << " Directory Bytes: " << _segments * _buckets * SIZEOF_DIR << std::endl; + std::cout << " Segments: " << _segments << std::endl; + std::cout << " Buckets per segment: " << _buckets << std::endl; + std::cout << " Entries: " << _segments * _buckets * DIR_DEPTH << std::endl; + int fd = _span->_fd; + //read directory + pread(fd, raw_dir, vol_dirlen(this)-vol_headerlen(this)-ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)), vol_headerlen(this)); + for (int s = 0; s < _segments; s++) { + CacheDirEntry *seg = dir_segment(s, this); + int seg_chain_max = 0; + int seg_empty = 0; + int seg_in_use = 0; + int seg_stale = 0; + int seg_bytes_in_use = 0; + int seg_dups = 0; + int seg_buckets_in_use = 0; + + ink_zero(chain_tag); + memset(chain_mark, -1, sizeof(chain_mark)); + for (int b = 0; b < _buckets; b++) { + CacheDirEntry *root = dir_bucket(b, seg); + int h = 0; + int chain_idx = 0; + int mark = 0; + ++seg_buckets_in_use; + // walking through the directories + for (CacheDirEntry *e = root; e; e = next_dir(e, seg)) { + if (!dir_offset(e)) { + ++seg_empty; + --seg_buckets_in_use; + // this should only happen on the first dir in a bucket + assert(nullptr == next_dir(e, seg)); + break; + } else { + int e_idx = e - seg; + ++h; + chain_tag[chain_idx++] = dir_tag(e); + if (chain_mark[e_idx] == mark) { + printf(" - Cycle of length %d detected for bucket %d\n", h, b); + } else if (chain_mark[e_idx] >= 0) { + printf(" - Entry %d is in chain %d and %d", e_idx, chain_mark[e_idx], mark); + } else { + chain_mark[e_idx] = mark; + } + + if (!dir_valid(this, e)) { + ++seg_stale; + } else { + uint64_t size = dir_approx_size(e); + if (dir_head(e)) { + ++head; + } + ++seg_in_use; + seg_bytes_in_use += size; + ++frag_demographics[dir_size(e)][dir_big(e)]; + } + } + e = next_dir(e, seg); + if (!e) { + break; + } + } + + // Check for duplicates (identical tags in the same bucket). + if (h > 1) { + unsigned short last; + qsort(chain_tag, h, sizeof(chain_tag[0]), &compare_ushort); + last = chain_tag[0]; + for (int k = 1; k < h; ++k) { + if (last == chain_tag[k]) { + ++seg_dups; + } + last = chain_tag[k]; + } + } + ++hist[std::min(h, SEGMENT_HISTOGRAM_WIDTH)]; + seg_chain_max = std::max(seg_chain_max, h); + } + int fl_size = dir_freelist_length(this, s); + in_use += seg_in_use; + empty += seg_empty; + stale += seg_stale; + free += fl_size; + buckets_in_use += seg_buckets_in_use; + max_chain_length = std::max(max_chain_length, seg_chain_max); + bytes_in_use += seg_bytes_in_use; + + printf(" - Segment-%d | Entries: used=%d stale=%d free=%d disk-bytes=%d Buckets: used=%d empty=%d max=%d avg=%.2f dups=%d\n", + s, seg_in_use, seg_stale, fl_size, seg_bytes_in_use, seg_buckets_in_use, seg_empty, seg_chain_max, + seg_buckets_in_use ? static_cast(seg_in_use + seg_stale) / seg_buckets_in_use : 0.0, seg_dups); + } +} + Errata Stripe::loadMeta() { @@ -303,6 +634,8 @@ Stripe::loadMeta() n.assign(pread(fd, stripe_buff, SBSIZE, pos)); data.setView(stripe_buff, n); meta = data.template at_ptr(0); + // TODO:: We need to read more data at this point to populate dir + dir = data.template at_ptr(vol_headerlen(this)); if (this->validateMeta(meta)) { delta = Bytes(data.template at_ptr(0) - stripe_buff); _meta[A][HEAD] = *meta; @@ -517,7 +850,7 @@ struct Cache { std::list _spans; std::map _volumes; std::vector globalVec_stripe; - std::unordered_set URLset; + std::unordered_set URLset; unsigned short *stripes_hash_table; }; @@ -837,9 +1170,13 @@ Cache::loadURLs(FilePath const &path) } else if (0 == strcasecmp(tag, TAG_VOL)) { std::string url; url.assign(blob.begin(), blob.size()); - int port = parser.getPort(url); - std::cout << "port # " << port << std::endl; - URLset.insert(url); + int port_ptr = -1, port_len = -1; + int port = parser.getPort(url, port_ptr, port_len); + if (port_ptr >= 0 && port_len > 0) + url.erase(port_ptr, port_len + 1); // get rid of :PORT + std::cout << "port # " << port << ":" << port_ptr << ":" << port_len << ":" << url << std::endl; + ts::CacheURL *curl = new ts::CacheURL(url, port); + URLset.emplace(curl); } } } else { @@ -1323,7 +1660,8 @@ VolumeConfig::load(FilePath const &path) } /* --------------------------------------------------------------------------------------- */ struct option Options[] = {{"help", 0, nullptr, 'h'}, {"spans", 1, nullptr, 's'}, {"volumes", 1, nullptr, 'v'}, - {"write", 0, nullptr, 'w'}, {"input", 1, nullptr, 'i'}, {nullptr, 0, nullptr, 0}}; + {"write", 0, nullptr, 'w'}, {"input", 1, nullptr, 'i'}, {"device", 1, nullptr, 'd'}, + {nullptr, 0, nullptr, 0}}; } Errata @@ -1396,22 +1734,15 @@ Find_Stripe(FilePath const &input_file_path) // scheme=http user=u password=p host=172.28.56.109 path=somepath query=somequery port=1234 // input file format: scheme://hostname:port/somepath;params?somequery user=USER password=PASS // user, password, are optional; scheme and host are required - MD5Context ctx; - INK_MD5 hashT; char hashStr[33]; - char *h = "https://ss.ss.@@s.ss:900/jhfjhgsfgnn"; - // char* h= "http://:@172.28.56.109/file7;?"port <== this is the format we need - url_matcher matcher; - if (matcher.match(h)) - std::cout << h << " : is valid" << std::endl; - else - std::cout << h << " : is NOT valid" << std::endl; - ctx.update(h, strlen(h)); - in_port_t port = 5005; - ctx.update(&port, sizeof(port)); - ctx.finalize(hashT); - printf("hash is %s: \n", ink_code_to_hex_str(hashStr, (unsigned char *)&hashT)); + // char* h= http://user:pass@IPADDRESS/path_to_file;?port <== this is the format we need + // url_matcher matcher; + // if (matcher.match(h)) + // std::cout << h << " : is valid" << std::endl; + // else + // std::cout << h << " : is NOT valid" << std::endl; + Errata zret; Cache cache; if (input_file_path) @@ -1421,51 +1752,48 @@ Find_Stripe(FilePath const &input_file_path) cache.dumpSpans(Cache::SpanDumpDepth::SPAN); cache.build_stripe_hash_table(); for (auto host : cache.URLset) { - INK_MD5 hash; - ink_code_md5((unsigned char *)host.data(), host.size(), (unsigned char *)&hash); - Stripe *stripe_ = cache.key_to_stripe(&hash, host.data(), host.size()); - printf("hash of %.*s is %s: Stripe %s \n", (int)host.size(), host.data(), - ink_code_to_hex_str(hashStr, (unsigned char *)&hash), stripe_->hashText.data()); + MD5Context ctx; + INK_MD5 hashT; + ctx.update(host->url.data(), host->url.size()); + ctx.update(&host->port, sizeof(host->port)); + ctx.finalize(hashT); + Stripe *stripe_ = cache.key_to_stripe(&hashT, host->url.data(), host->url.size()); + printf("hash of %.*s is %s: Stripe %s \n", (int)host->url.size(), host->url.data(), + ink_code_to_hex_str(hashStr, (unsigned char *)&hashT), stripe_->hashText.data()); } } return zret; } - -bool -ts::URLparser::verifyURL(std::string &url1) +Errata +dir_check() { + Errata zret; + Cache cache; + if ((zret = cache.loadSpan(SpanFile))) { + cache.dumpSpans(Cache::SpanDumpDepth::SPAN); + for (auto &stripe : cache.globalVec_stripe) { + stripe->dir_check(); + } + } + return zret; } -int -ts::URLparser::getPort(std::string &fullURL) -{ - url_matcher matcher; - static const ts::StringView HTTP("http"); - static const ts::StringView HTTPS("https"); - ts::StringView url(fullURL.data(), (int)fullURL.size()); - // check for scheme - ts::StringView scheme = url.splitPrefix(':'); - if ((strcasecmp(scheme, HTTP) == 0) || (strcasecmp(scheme, HTTPS) == 0)) { - url += 2; - ts::StringView hostPort = url.splitPrefix(':'); - if (hostPort) // i.e. port is present - { - ts::StringView port = url.splitPrefix('/'); - if (!port) // i.e. backslash is not present, then the rest of url must be just port - port = url; - if (matcher.portmatch(port.begin(), port.size())) { - ts::StringView text; - auto n = ts::svtoi(port, &text); - if (text == port) - return n; +Errata +Clear_Span(std::string devicePath) +{ + Errata zret; + Cache cache; + if ((zret = cache.loadSpan(SpanFile))) { + cache.dumpSpans(Cache::SpanDumpDepth::SPAN); + for (auto sp : cache._spans) { + if (0 == strncmp(sp->_path.path(), devicePath.data(), devicePath.size())) { + printf("clearing %s\n", devicePath.data()); + sp->clearPermanently(); } } - return -1; - } else { - std::cout << "No scheme provided for: " << scheme.begin() << std::endl; - return -1; } + return zret; } int @@ -1475,6 +1803,7 @@ main(int argc, char *argv[]) int opt_val; bool help = false; FilePath input_url_file; + std::string inputFile; while (-1 != (opt_val = getopt_long(argc, argv, "h", Options, &opt_idx))) { switch (opt_val) { case 'h': @@ -1494,6 +1823,10 @@ main(int argc, char *argv[]) case 'i': input_url_file = optarg; break; + case 'd': + char *inp = strdup(optarg); + inputFile.assign(inp, strlen(inp)); + break; } } @@ -1501,11 +1834,14 @@ main(int argc, char *argv[]) .subCommand(std::string("stripes"), std::string("List the stripes"), []() { return List_Stripes(Cache::SpanDumpDepth::STRIPE); }); Commands.add(std::string("clear"), std::string("Clear spans"), &Clear_Spans); + Commands.add(std::string("check"), std::string("cache check"), &dir_check); Commands.add(std::string("volumes"), std::string("Volumes"), &Simulate_Span_Allocation); Commands.add(std::string("alloc"), std::string("Storage allocation")) .subCommand(std::string("free"), std::string("Allocate storage on free (empty) spans"), &Cmd_Allocate_Empty_Spans); Commands.add(std::string("find"), std::string("Find Stripe Assignment")) .subCommand(std::string("url"), std::string("URL"), [&](int, char *argv[]) { return Find_Stripe(input_url_file); }); + Commands.add(std::string("clearspan"), std::string("clear specific span")) + .subCommand(std::string("span"), std::string("device path"), [&](int, char *argv[]) { return Clear_Span(inputFile); }); Commands.setArgIndex(optind); diff --git a/cmd/traffic_cache_tool/Makefile.am b/cmd/traffic_cache_tool/Makefile.am index 129e43c7584..e0d4ac8afdf 100644 --- a/cmd/traffic_cache_tool/Makefile.am +++ b/cmd/traffic_cache_tool/Makefile.am @@ -21,7 +21,7 @@ AM_CPPFLAGS = -I $(srcdir)/lib/ts noinst_PROGRAMS = traffic_cache_tool -traffic_cache_tool_SOURCES = CacheDefs.h CacheTool.cc File.h File.cc Command.h Command.cc +traffic_cache_tool_SOURCES = CacheDefs.h CacheDefs.cc CacheTool.cc File.h File.cc Command.h Command.cc traffic_cache_tool_LDADD = \ $(top_builddir)/lib/ts/.libs/MemView.o \ $(top_builddir)/lib/ts/.libs/ink_memory.o \ diff --git a/iocore/cache/CacheDir.cc b/iocore/cache/CacheDir.cc index 2616d818fcb..9833da37d49 100644 --- a/iocore/cache/CacheDir.cc +++ b/iocore/cache/CacheDir.cc @@ -1209,7 +1209,7 @@ int Vol::dir_check(bool /* fix ATS_UNUSED */) // TODO: we should eliminate this ink_zero(frag_demographics); - printf("Stripe '[%s]'\n", hash_text.get()); + printf(" Stripe '[%s]'\n", hash_text.get()); printf(" Directory Bytes: %" PRIu64 "\n", total_buckets * SIZEOF_DIR); printf(" Segments: %d\n", segments); printf(" Buckets per segment: %" PRIu64 "\n", buckets); diff --git a/proxy/hdrs/URL.cc b/proxy/hdrs/URL.cc index 8d10430f46d..1eb5061255a 100644 --- a/proxy/hdrs/URL.cc +++ b/proxy/hdrs/URL.cc @@ -1757,8 +1757,6 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca if (p == e) { char hashStr[33]; - Debug("cache", "p==e string to be hashed: %s%d%d => %s", buffer, port, generation, - ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); ctx.update(buffer, BUFSIZE); p = buffer; } @@ -1768,8 +1766,6 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca if (p != buffer) { char hashStr[33]; - Debug("cache", "p!=buffer string to be hashed: %s%d%d => %s", buffer, port, generation, - ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); ctx.update(buffer, p - buffer); } @@ -1781,7 +1777,7 @@ url_MD5_get_general(const URLImpl *url, CryptoContext &ctx, CryptoHash &hash, ca } ctx.finalize(hash); char hashStr[33]; - Debug("cache", "string to be hashed: %s%d%d => %s", buffer, port, generation, + Debug("cache", "string to be hashed: %.*s%d%d => %s", (p - buffer), buffer, port, generation, ink_code_to_hex_str(hashStr, (unsigned char *)&hash)); } From 19ff098a0099ca9326dbd43e921d652af0a6cd7e Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Fri, 10 Nov 2017 17:10:14 -0600 Subject: [PATCH 15/23] vol header len --- cmd/traffic_cache_tool/CacheTool.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 432b26371dc..c2b36f668ef 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -282,14 +282,14 @@ Stripe::updateLiveData(enum Copy c) TS_INLINE int vol_headerlen(Stripe *d) { - return ROUND_TO_STORE_BLOCK(sizeof(CacheDirEntry) + sizeof(uint16_t) * (d->_segments - 1)); + return ROUND_TO_STORE_BLOCK(sizeof(StripeMeta) + sizeof(uint16_t) * (d->_segments - 1)); } size_t vol_dirlen(Stripe *d) { return vol_headerlen(d) + ROUND_TO_STORE_BLOCK(((size_t)d->_buckets) * DIR_DEPTH * d->_segments * SIZEOF_DIR) + - ROUND_TO_STORE_BLOCK(sizeof(CacheDirEntry)); + ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)); } #define dir_big(_e) ((uint32_t)((((_e)->w[1]) >> 8) & 0x3)) @@ -464,7 +464,7 @@ dir_freelist_length(Stripe *d, int s) int free = 0; CacheDirEntry *seg = dir_segment(s, d); //TODO: check freelist[s]; ATS passes s to freelist, I don't know how that works -_- - CacheDirEntry *e = dir_from_offset(d->_meta[0][0].freelist[0], seg); + CacheDirEntry *e = dir_from_offset(d->_meta[0][0].freelist[s], seg); if (dir_bucket_loop_fix(e, s, d)) { return (DIR_DEPTH - 1) * d->_buckets; } @@ -490,7 +490,7 @@ Stripe::dir_check() int32_t chain_mark[MAX_ENTRIES_PER_SEGMENT]; this->loadMeta(); - // create raw_dir; + // create raw_dir pointing at the first ever dir in the stripe; char *raw_dir = (char *)ats_memalign(ats_pagesize(), vol_dirlen(this)-vol_headerlen(this)-ROUND_TO_STORE_BLOCK(sizeof(StripeMeta))); dir = (CacheDirEntry *)raw_dir; uint64_t total_buckets = _segments * _buckets; From 26ceff26aa460878982a22e7cd580d29cfc456c8 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Mon, 13 Nov 2017 17:36:55 -0600 Subject: [PATCH 16/23] copy freelist in a member variable --- cmd/traffic_cache_tool/CacheTool.cc | 33 ++++++++++++++++++----------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index c2b36f668ef..2b97d18d8a2 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -183,6 +183,7 @@ struct Stripe { /// Directory. Chunk _directory; CacheDirEntry const *dir = nullptr; + uint16_t *freelist = nullptr; }; Stripe::Chunk::~Chunk() @@ -319,7 +320,7 @@ vol_dirlen(Stripe *d) TS_INLINE CacheDirEntry * vol_dir_segment(Stripe *d, int s) { - return (CacheDirEntry *)((d->dir + d->_directory._skip) + (s * d->_buckets) * DIR_DEPTH * SIZEOF_DIR); + return (CacheDirEntry *)((d->dir) + (s * d->_buckets) * DIR_DEPTH * SIZEOF_DIR); } TS_INLINE CacheDirEntry * @@ -463,8 +464,8 @@ dir_freelist_length(Stripe *d, int s) { int free = 0; CacheDirEntry *seg = dir_segment(s, d); - //TODO: check freelist[s]; ATS passes s to freelist, I don't know how that works -_- - CacheDirEntry *e = dir_from_offset(d->_meta[0][0].freelist[s], seg); + // TODO: check freelist[s]; ATS passes s to freelist, I don't know how that works -_- + CacheDirEntry *e = dir_from_offset(d->freelist[s], seg); if (dir_bucket_loop_fix(e, s, d)) { return (DIR_DEPTH - 1) * d->_buckets; } @@ -491,8 +492,8 @@ Stripe::dir_check() this->loadMeta(); // create raw_dir pointing at the first ever dir in the stripe; - char *raw_dir = (char *)ats_memalign(ats_pagesize(), vol_dirlen(this)-vol_headerlen(this)-ROUND_TO_STORE_BLOCK(sizeof(StripeMeta))); - dir = (CacheDirEntry *)raw_dir; + char *raw_dir = (char *)ats_memalign(ats_pagesize(), vol_dirlen(this)); + dir = reinterpret_cast(raw_dir + vol_headerlen(this)); uint64_t total_buckets = _segments * _buckets; uint64_t total_entries = total_buckets * DIR_DEPTH; int frag_demographics[1 << DIR_SIZE_WIDTH][DIR_BLOCK_SIZES]; @@ -508,8 +509,8 @@ Stripe::dir_check() std::cout << " Buckets per segment: " << _buckets << std::endl; std::cout << " Entries: " << _segments * _buckets * DIR_DEPTH << std::endl; int fd = _span->_fd; - //read directory - pread(fd, raw_dir, vol_dirlen(this)-vol_headerlen(this)-ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)), vol_headerlen(this)); + // read directory + pread(fd, raw_dir, vol_dirlen(this), this->_start); for (int s = 0; s < _segments; s++) { CacheDirEntry *seg = dir_segment(s, this); int seg_chain_max = 0; @@ -622,7 +623,7 @@ Stripe::loadMeta() std::unique_ptr bulk_buff; // Buffer for bulk reads. static const size_t SBSIZE = CacheStoreBlocks::SCALE; // save some typing. alignas(SBSIZE) char stripe_buff[SBSIZE]; // Use when reading a single stripe block. - + alignas(SBSIZE) char stripe_buff2[SBSIZE]; // use to save the stripe freelist if (io_align > SBSIZE) return Errata::Message(0, 1, "Cannot load stripe ", _idx, " on span ", _span->_path, " because the I/O block alignment ", io_align, " is larger than the buffer alignment ", SBSIZE); @@ -631,13 +632,13 @@ Stripe::loadMeta() // Header A must be at the start of the stripe block. // Todo: really need to check pread() for failure. - n.assign(pread(fd, stripe_buff, SBSIZE, pos)); - data.setView(stripe_buff, n); + ssize_t headerbyteCount = pread(fd, stripe_buff2, SBSIZE, pos); + n.assign(headerbyteCount); + data.setView(stripe_buff2, n); meta = data.template at_ptr(0); // TODO:: We need to read more data at this point to populate dir - dir = data.template at_ptr(vol_headerlen(this)); if (this->validateMeta(meta)) { - delta = Bytes(data.template at_ptr(0) - stripe_buff); + delta = Bytes(data.template at_ptr(0) - stripe_buff2); _meta[A][HEAD] = *meta; _meta_pos[A][HEAD] = round_down(pos + Bytes(delta)); pos += round_up(SBSIZE); @@ -710,6 +711,14 @@ Stripe::loadMeta() } } + n.assign(headerbyteCount); + data.setView(stripe_buff2, n); + meta = data.template at_ptr(0); + // copy freelist + freelist = (uint16_t *)malloc(_segments * sizeof(uint16_t)); + for (int i = 0; i < _segments; i++) + freelist[i] = meta->freelist[i]; + if (!zret) _directory.clear(); return zret; From 3fb6be10272acc43a9adf11aaf979fb4ce023a9a Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Tue, 14 Nov 2017 17:49:45 -0600 Subject: [PATCH 17/23] use ATS calculation to calculate bucket --- cmd/traffic_cache_tool/CacheTool.cc | 63 +++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 2b97d18d8a2..d455c489d2d 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -54,7 +54,9 @@ using ts::FilePath; using ts::MemView; using ts::CacheDirEntry; +constexpr int ESTIMATED_OBJECT_SIZE = 8000; constexpr int VOL_HASH_TABLE_SIZE = 32707; +int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; CacheStoreBlocks Vol_hash_alloc_size(1024); constexpr unsigned short VOL_HASH_EMPTY = 65535; constexpr int DIR_TAG_WIDTH = 12; @@ -163,7 +165,7 @@ struct Stripe { Span *_span; ///< Hosting span. INK_MD5 hash_id; /// hash_id - Bytes _start; ///< Offset of first byte of stripe. + Bytes _start; ///< Offset of first byte of stripe metadata. Bytes _content; ///< Start of content. CacheStoreBlocks _len; ///< Length of stripe. uint8_t _vol_idx = 0; ///< Volume index. @@ -249,6 +251,44 @@ Stripe::probeMeta(MemView &mem, StripeMeta const *base_meta) return false; } +/* INK_ALIGN() is only to be used to align on a power of 2 boundary */ +#define INK_ALIGN(size, boundary) (((size) + ((boundary)-1)) & ~((boundary)-1)) + +#define ROUND_TO_STORE_BLOCK(_x) INK_ALIGN((_x), 8192) + +TS_INLINE int +vol_headerlen(Stripe *d) +{ + return ROUND_TO_STORE_BLOCK(sizeof(StripeMeta) + sizeof(uint16_t) * (d->_segments - 1)); +} + +size_t +vol_dirlen(Stripe *d) +{ + return vol_headerlen(d) + ROUND_TO_STORE_BLOCK(((size_t)d->_buckets) * DIR_DEPTH * d->_segments * SIZEOF_DIR) + + ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)); +} + + +static void +vol_init_data_internal(Stripe *d) +{ + int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; + d->_buckets = ((d->_len.count()*8192 - (d->_content - d->_start)) / cache_config_min_average_object_size) / DIR_DEPTH; + d->_segments = (d->_buckets + (((1 << 16) - 1) / DIR_DEPTH)) / ((1 << 16) / DIR_DEPTH); + d->_buckets = (d->_buckets + d->_segments - 1) / d->_segments; + d->_content = d->_start + Bytes(2 * vol_dirlen(d)); +} + +static void +vol_init_data(Stripe *d) +{ + // iteratively calculate start + buckets + vol_init_data_internal(d); + vol_init_data_internal(d); + vol_init_data_internal(d); +} + void Stripe::updateLiveData(enum Copy c) { @@ -257,6 +297,10 @@ Stripe::updateLiveData(enum Copy c) int64_t n_buckets; int64_t n_segments; + _content = _start; + vol_init_data(this); + /* + * COMMENTING THIS SECTION FOR NOW TO USE THE EXACT LOGIN USED IN ATS TO CALCULATE THE NUMBER OF SEGMENTS AND BUCKETS // Past the header is the segment free list heads which if sufficiently long (> ~4K) can take // more than 1 store block. Start with a guess of 1 and adjust upwards as needed. A 2TB stripe // with an AOS of 8000 has roughly 3700 segments meaning that for even 10TB drives this loop @@ -272,26 +316,11 @@ Stripe::updateLiveData(enum Copy c) _buckets = n_buckets / n_segments; _segments = n_segments; + */ _directory._skip = header_len; } -/* INK_ALIGN() is only to be used to align on a power of 2 boundary */ -#define INK_ALIGN(size, boundary) (((size) + ((boundary)-1)) & ~((boundary)-1)) - -#define ROUND_TO_STORE_BLOCK(_x) INK_ALIGN((_x), 8192) -TS_INLINE int -vol_headerlen(Stripe *d) -{ - return ROUND_TO_STORE_BLOCK(sizeof(StripeMeta) + sizeof(uint16_t) * (d->_segments - 1)); -} - -size_t -vol_dirlen(Stripe *d) -{ - return vol_headerlen(d) + ROUND_TO_STORE_BLOCK(((size_t)d->_buckets) * DIR_DEPTH * d->_segments * SIZEOF_DIR) + - ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)); -} #define dir_big(_e) ((uint32_t)((((_e)->w[1]) >> 8) & 0x3)) #define dir_bit(_e, _w, _b) ((uint32_t)(((_e)->w[_w] >> (_b)) & 1)) From 3c7cdf9e10335a9f5ed7b7c0b1184a28b23ef783 Mon Sep 17 00:00:00 2001 From: Syeda Persia Aziz Date: Tue, 14 Nov 2017 23:06:37 -0600 Subject: [PATCH 18/23] dir_check worksgit add -u (well almost) --- cmd/traffic_cache_tool/CacheTool.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index d455c489d2d..40cbf84bedd 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -349,7 +349,7 @@ Stripe::updateLiveData(enum Copy c) TS_INLINE CacheDirEntry * vol_dir_segment(Stripe *d, int s) { - return (CacheDirEntry *)((d->dir) + (s * d->_buckets) * DIR_DEPTH * SIZEOF_DIR); + return (CacheDirEntry *)(((char *)d->dir) + (s * d->_buckets) * DIR_DEPTH * SIZEOF_DIR); } TS_INLINE CacheDirEntry * @@ -522,7 +522,7 @@ Stripe::dir_check() this->loadMeta(); // create raw_dir pointing at the first ever dir in the stripe; char *raw_dir = (char *)ats_memalign(ats_pagesize(), vol_dirlen(this)); - dir = reinterpret_cast(raw_dir + vol_headerlen(this)); + dir = (CacheDirEntry *)(raw_dir + vol_headerlen(this)); uint64_t total_buckets = _segments * _buckets; uint64_t total_entries = total_buckets * DIR_DEPTH; int frag_demographics[1 << DIR_SIZE_WIDTH][DIR_BLOCK_SIZES]; From 5b00b19b207621326fc281569ffe366a609e5e48 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Wed, 15 Nov 2017 10:52:06 -0600 Subject: [PATCH 19/23] Dir_check fully worksgit add -u yaya --- cmd/traffic_cache_tool/CacheTool.cc | 97 ++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 22 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 40cbf84bedd..4a79770a3b6 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -42,6 +42,7 @@ #include #include #include +#include using ts::Bytes; using ts::Megabytes; @@ -54,9 +55,9 @@ using ts::FilePath; using ts::MemView; using ts::CacheDirEntry; -constexpr int ESTIMATED_OBJECT_SIZE = 8000; -constexpr int VOL_HASH_TABLE_SIZE = 32707; -int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; +constexpr int ESTIMATED_OBJECT_SIZE = 8000; +constexpr int VOL_HASH_TABLE_SIZE = 32707; +int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; CacheStoreBlocks Vol_hash_alloc_size(1024); constexpr unsigned short VOL_HASH_EMPTY = 65535; constexpr int DIR_TAG_WIDTH = 12; @@ -186,6 +187,9 @@ struct Stripe { Chunk _directory; CacheDirEntry const *dir = nullptr; uint16_t *freelist = nullptr; + int dir_freelist_length(int s); + TS_INLINE CacheDirEntry *dir_segment(int s); + TS_INLINE CacheDirEntry *vol_dir_segment(int s); }; Stripe::Chunk::~Chunk() @@ -269,15 +273,14 @@ vol_dirlen(Stripe *d) ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)); } - static void vol_init_data_internal(Stripe *d) { - int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; - d->_buckets = ((d->_len.count()*8192 - (d->_content - d->_start)) / cache_config_min_average_object_size) / DIR_DEPTH; + int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; + d->_buckets = ((d->_len.count() * 8192 - (d->_content - d->_start)) / cache_config_min_average_object_size) / DIR_DEPTH; d->_segments = (d->_buckets + (((1 << 16) - 1) / DIR_DEPTH)) / ((1 << 16) / DIR_DEPTH); d->_buckets = (d->_buckets + d->_segments - 1) / d->_segments; - d->_content = d->_start + Bytes(2 * vol_dirlen(d)); + d->_content = d->_start + Bytes(2 * vol_dirlen(d)); } static void @@ -320,8 +323,6 @@ Stripe::updateLiveData(enum Copy c) _directory._skip = header_len; } - - #define dir_big(_e) ((uint32_t)((((_e)->w[1]) >> 8) & 0x3)) #define dir_bit(_e, _w, _b) ((uint32_t)(((_e)->w[_w] >> (_b)) & 1)) #define dir_size(_e) ((uint32_t)(((_e)->w[1]) >> 10)) @@ -337,7 +338,7 @@ Stripe::updateLiveData(enum Copy c) (_e)->w[1] = (uint16_t)((((_o) >> 16) & 0xFF) | ((_e)->w[1] & 0xFF00)); \ (_e)->w[4] = (uint16_t)((_o) >> 24); \ } while (0) -#define dir_segment(_s, _d) vol_dir_segment(_d, _s) +//#define dir_segment(_s, _d) vol_dir_segment(_d, _s) #define dir_in_seg(_s, _i) ((CacheDirEntry *)(((char *)(_s)) + (SIZEOF_DIR * (_i)))) #define dir_next(_e) (_e)->w[3] #define dir_phase(_e) dir_bit(_e, 2, 12) @@ -347,9 +348,15 @@ Stripe::updateLiveData(enum Copy c) #define dir_set_next(_e, _o) (_e)->w[3] = (uint16_t)(_o) TS_INLINE CacheDirEntry * -vol_dir_segment(Stripe *d, int s) +Stripe::dir_segment(int s) +{ + return vol_dir_segment(s); +} + +TS_INLINE CacheDirEntry * +Stripe::vol_dir_segment(int s) { - return (CacheDirEntry *)(((char *)d->dir) + (s * d->_buckets) * DIR_DEPTH * SIZEOF_DIR); + return (CacheDirEntry *)(((char *)this->dir) + (s * this->_buckets) * DIR_DEPTH * SIZEOF_DIR); } TS_INLINE CacheDirEntry * @@ -413,7 +420,7 @@ dir_to_offset(const CacheDirEntry *d, const CacheDirEntry *seg) void dir_free_entry(CacheDirEntry *e, int s, Stripe *d) { - CacheDirEntry *seg = dir_segment(s, d); + CacheDirEntry *seg = d->dir_segment(s); unsigned int fo = d->_meta[0][0].freelist[s]; unsigned int eo = dir_to_offset(e, seg); dir_set_next(e, fo); @@ -429,7 +436,7 @@ void dir_init_segment(int s, Stripe *d) { d->_meta[0][0].freelist[s] = 0; - CacheDirEntry *seg = dir_segment(s, d); + CacheDirEntry *seg = d->dir_segment(s); int l, b; memset(seg, 0, SIZEOF_DIR * DIR_DEPTH * d->_buckets); for (l = 1; l < DIR_DEPTH; l++) { @@ -481,7 +488,7 @@ dir_bucket_loop_check(CacheDirEntry *start_dir, CacheDirEntry *seg) int dir_bucket_loop_fix(CacheDirEntry *start_dir, int s, Stripe *d) { - if (!dir_bucket_loop_check(start_dir, dir_segment(s, d))) { + if (!dir_bucket_loop_check(start_dir, d->dir_segment(s))) { dir_init_segment(s, d); return 1; } @@ -489,14 +496,14 @@ dir_bucket_loop_fix(CacheDirEntry *start_dir, int s, Stripe *d) } int -dir_freelist_length(Stripe *d, int s) +Stripe::dir_freelist_length(int s) { int free = 0; - CacheDirEntry *seg = dir_segment(s, d); + CacheDirEntry *seg = this->dir_segment(s); // TODO: check freelist[s]; ATS passes s to freelist, I don't know how that works -_- - CacheDirEntry *e = dir_from_offset(d->freelist[s], seg); - if (dir_bucket_loop_fix(e, s, d)) { - return (DIR_DEPTH - 1) * d->_buckets; + CacheDirEntry *e = dir_from_offset(this->freelist[s], seg); + if (dir_bucket_loop_fix(e, s, this)) { + return (DIR_DEPTH - 1) * this->_buckets; } while (e) { free++; @@ -541,7 +548,7 @@ Stripe::dir_check() // read directory pread(fd, raw_dir, vol_dirlen(this), this->_start); for (int s = 0; s < _segments; s++) { - CacheDirEntry *seg = dir_segment(s, this); + CacheDirEntry *seg = this->dir_segment(s); int seg_chain_max = 0; int seg_empty = 0; int seg_in_use = 0; @@ -611,7 +618,7 @@ Stripe::dir_check() ++hist[std::min(h, SEGMENT_HISTOGRAM_WIDTH)]; seg_chain_max = std::max(seg_chain_max, h); } - int fl_size = dir_freelist_length(this, s); + int fl_size = dir_freelist_length(s); in_use += seg_in_use; empty += seg_empty; stale += seg_stale; @@ -624,6 +631,51 @@ Stripe::dir_check() s, seg_in_use, seg_stale, fl_size, seg_bytes_in_use, seg_buckets_in_use, seg_empty, seg_chain_max, seg_buckets_in_use ? static_cast(seg_in_use + seg_stale) / seg_buckets_in_use : 0.0, seg_dups); } + ////////////////// + + printf(" - Stripe | Entries: in-use=%d stale=%d free=%d Buckets: empty=%d max=%d avg=%.2f\n", in_use, stale, free, empty, + max_chain_length, buckets_in_use ? static_cast(in_use + stale) / buckets_in_use : 0); + + printf(" Chain lengths: "); + for (j = 0; j < SEGMENT_HISTOGRAM_WIDTH; ++j) { + printf(" %d=%d ", j, hist[j]); + } + printf(" %d>=%d\n", SEGMENT_HISTOGRAM_WIDTH, hist[SEGMENT_HISTOGRAM_WIDTH]); + + char tt[256]; + printf(" Total Size: %" PRIu64 "\n", static_cast(_len.count())); + printf(" Bytes in Use: %" PRIu64 " [%0.2f%%]\n", bytes_in_use, 100.0 * (static_cast(bytes_in_use) / _len.count())); + printf(" Objects: %d\n", head); + printf(" Average Size: %" PRIu64 "\n", head ? (bytes_in_use / head) : 0); + printf(" Average Frags: %.2f\n", head ? static_cast(in_use) / head : 0); + printf(" Write Position: %" PRIu64 "\n", _meta[0][0].write_pos - _content.count()); + printf(" Wrap Count: %d\n", _meta[0][0].cycle); + printf(" Phase: %s\n", _meta[0][0].phase ? "true" : "false"); + ctime_r(&_meta[0][0].create_time, tt); + tt[strlen(tt) - 1] = 0; + printf(" Sync Serial: %u\n", _meta[0][0].sync_serial); + printf(" Write Serial: %u\n", _meta[0][0].write_serial); + printf(" Create Time: %s\n", tt); + printf("\n"); + printf(" Fragment size demographics\n"); + for (int b = 0; b < DIR_BLOCK_SIZES; ++b) { + int block_size = DIR_BLOCK_SIZE(b); + int s = 0; + while (s < 1 << DIR_SIZE_WIDTH) { + for (int j = 0; j < 8; ++j, ++s) { + // The size markings are redundant. Low values (less than DIR_SHIFT_WIDTH) for larger + // base block sizes should never be used. Such entries should use the next smaller base block size. + if (b > 0 && s < 1 << DIR_BLOCK_SHIFT(1)) { + assert(frag_demographics[s][b] == 0); + continue; + } + printf(" %8d[%2d:%1d]:%06d", (s + 1) * block_size, s, b, frag_demographics[s][b]); + } + printf("\n"); + } + } + printf("\n"); + //////////////// } Errata @@ -1814,6 +1866,7 @@ dir_check() stripe->dir_check(); } } + printf("\nCHECK succeeded\n"); return zret; } From c37ede05c7db292fff72e52dbc54bf99143e6650 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Wed, 15 Nov 2017 14:31:12 -0600 Subject: [PATCH 20/23] refactoring work --- cmd/traffic_cache_tool/CacheTool.cc | 74 ++++++++++++++++------------- 1 file changed, 41 insertions(+), 33 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 4a79770a3b6..7822dd09290 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -190,6 +190,13 @@ struct Stripe { int dir_freelist_length(int s); TS_INLINE CacheDirEntry *dir_segment(int s); TS_INLINE CacheDirEntry *vol_dir_segment(int s); + size_t vol_dirlen(); + TS_INLINE int vol_headerlen(); + void vol_init_data_internal(); + void vol_init_data(); + int dir_bucket_loop_fix(CacheDirEntry *start_dir, int s); + void dir_init_segment(int s); + void dir_free_entry(CacheDirEntry *e, int s); }; Stripe::Chunk::~Chunk() @@ -261,35 +268,36 @@ Stripe::probeMeta(MemView &mem, StripeMeta const *base_meta) #define ROUND_TO_STORE_BLOCK(_x) INK_ALIGN((_x), 8192) TS_INLINE int -vol_headerlen(Stripe *d) +Stripe::vol_headerlen() { - return ROUND_TO_STORE_BLOCK(sizeof(StripeMeta) + sizeof(uint16_t) * (d->_segments - 1)); + return ROUND_TO_STORE_BLOCK(sizeof(StripeMeta) + sizeof(uint16_t) * (this->_segments - 1)); } size_t -vol_dirlen(Stripe *d) +Stripe::vol_dirlen() { - return vol_headerlen(d) + ROUND_TO_STORE_BLOCK(((size_t)d->_buckets) * DIR_DEPTH * d->_segments * SIZEOF_DIR) + + return vol_headerlen() + ROUND_TO_STORE_BLOCK(((size_t)this->_buckets) * DIR_DEPTH * this->_segments * SIZEOF_DIR) + ROUND_TO_STORE_BLOCK(sizeof(StripeMeta)); } -static void -vol_init_data_internal(Stripe *d) +void +Stripe::vol_init_data_internal() { int cache_config_min_average_object_size = ESTIMATED_OBJECT_SIZE; - d->_buckets = ((d->_len.count() * 8192 - (d->_content - d->_start)) / cache_config_min_average_object_size) / DIR_DEPTH; - d->_segments = (d->_buckets + (((1 << 16) - 1) / DIR_DEPTH)) / ((1 << 16) / DIR_DEPTH); - d->_buckets = (d->_buckets + d->_segments - 1) / d->_segments; - d->_content = d->_start + Bytes(2 * vol_dirlen(d)); + this->_buckets = + ((this->_len.count() * 8192 - (this->_content - this->_start)) / cache_config_min_average_object_size) / DIR_DEPTH; + this->_segments = (this->_buckets + (((1 << 16) - 1) / DIR_DEPTH)) / ((1 << 16) / DIR_DEPTH); + this->_buckets = (this->_buckets + this->_segments - 1) / this->_segments; + this->_content = this->_start + Bytes(2 * vol_dirlen()); } -static void -vol_init_data(Stripe *d) +void +Stripe::vol_init_data() { // iteratively calculate start + buckets - vol_init_data_internal(d); - vol_init_data_internal(d); - vol_init_data_internal(d); + this->vol_init_data_internal(); + this->vol_init_data_internal(); + this->vol_init_data_internal(); } void @@ -301,7 +309,7 @@ Stripe::updateLiveData(enum Copy c) int64_t n_segments; _content = _start; - vol_init_data(this); + this->vol_init_data(); /* * COMMENTING THIS SECTION FOR NOW TO USE THE EXACT LOGIN USED IN ATS TO CALCULATE THE NUMBER OF SEGMENTS AND BUCKETS // Past the header is the segment free list heads which if sufficiently long (> ~4K) can take @@ -418,31 +426,31 @@ dir_to_offset(const CacheDirEntry *d, const CacheDirEntry *seg) } void -dir_free_entry(CacheDirEntry *e, int s, Stripe *d) +Stripe::dir_free_entry(CacheDirEntry *e, int s) { - CacheDirEntry *seg = d->dir_segment(s); - unsigned int fo = d->_meta[0][0].freelist[s]; + CacheDirEntry *seg = this->dir_segment(s); + unsigned int fo = this->_meta[0][0].freelist[s]; unsigned int eo = dir_to_offset(e, seg); dir_set_next(e, fo); if (fo) { dir_set_prev(dir_from_offset(fo, seg), eo); } - d->_meta[0][0].freelist[s] = eo; + this->_meta[0][0].freelist[s] = eo; } // adds all the directory entries // in a segment to the segment freelist void -dir_init_segment(int s, Stripe *d) +Stripe::dir_init_segment(int s) { - d->_meta[0][0].freelist[s] = 0; - CacheDirEntry *seg = d->dir_segment(s); + this->_meta[0][0].freelist[s] = 0; + CacheDirEntry *seg = this->dir_segment(s); int l, b; - memset(seg, 0, SIZEOF_DIR * DIR_DEPTH * d->_buckets); + memset(seg, 0, SIZEOF_DIR * DIR_DEPTH * this->_buckets); for (l = 1; l < DIR_DEPTH; l++) { - for (b = 0; b < d->_buckets; b++) { + for (b = 0; b < this->_buckets; b++) { CacheDirEntry *bucket = dir_bucket(b, seg); - dir_free_entry(dir_bucket_row(bucket, l), s, d); + this->dir_free_entry(dir_bucket_row(bucket, l), s); } } } @@ -486,10 +494,10 @@ dir_bucket_loop_check(CacheDirEntry *start_dir, CacheDirEntry *seg) // Note : abuse of the token bit in dir entries int -dir_bucket_loop_fix(CacheDirEntry *start_dir, int s, Stripe *d) +Stripe::dir_bucket_loop_fix(CacheDirEntry *start_dir, int s) { - if (!dir_bucket_loop_check(start_dir, d->dir_segment(s))) { - dir_init_segment(s, d); + if (!dir_bucket_loop_check(start_dir, this->dir_segment(s))) { + this->dir_init_segment(s); return 1; } return 0; @@ -502,7 +510,7 @@ Stripe::dir_freelist_length(int s) CacheDirEntry *seg = this->dir_segment(s); // TODO: check freelist[s]; ATS passes s to freelist, I don't know how that works -_- CacheDirEntry *e = dir_from_offset(this->freelist[s], seg); - if (dir_bucket_loop_fix(e, s, this)) { + if (this->dir_bucket_loop_fix(e, s)) { return (DIR_DEPTH - 1) * this->_buckets; } while (e) { @@ -528,8 +536,8 @@ Stripe::dir_check() this->loadMeta(); // create raw_dir pointing at the first ever dir in the stripe; - char *raw_dir = (char *)ats_memalign(ats_pagesize(), vol_dirlen(this)); - dir = (CacheDirEntry *)(raw_dir + vol_headerlen(this)); + char *raw_dir = (char *)ats_memalign(ats_pagesize(), this->vol_dirlen()); + dir = (CacheDirEntry *)(raw_dir + this->vol_headerlen()); uint64_t total_buckets = _segments * _buckets; uint64_t total_entries = total_buckets * DIR_DEPTH; int frag_demographics[1 << DIR_SIZE_WIDTH][DIR_BLOCK_SIZES]; @@ -546,7 +554,7 @@ Stripe::dir_check() std::cout << " Entries: " << _segments * _buckets * DIR_DEPTH << std::endl; int fd = _span->_fd; // read directory - pread(fd, raw_dir, vol_dirlen(this), this->_start); + pread(fd, raw_dir, this->vol_dirlen(), this->_start); for (int s = 0; s < _segments; s++) { CacheDirEntry *seg = this->dir_segment(s); int seg_chain_max = 0; From a49472f1ae36c7541792d035ab47f90395e98ffb Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Wed, 15 Nov 2017 17:07:14 -0600 Subject: [PATCH 21/23] fix critical error --- cmd/traffic_cache_tool/CacheTool.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 7822dd09290..5bf998bc000 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -1886,7 +1886,7 @@ Clear_Span(std::string devicePath) if ((zret = cache.loadSpan(SpanFile))) { cache.dumpSpans(Cache::SpanDumpDepth::SPAN); for (auto sp : cache._spans) { - if (0 == strncmp(sp->_path.path(), devicePath.data(), devicePath.size())) { + if (devicePath.size()>0 && 0 == strncmp(sp->_path.path(), devicePath.data(), devicePath.size())) { printf("clearing %s\n", devicePath.data()); sp->clearPermanently(); } From 6c632ea4e738151dd212590b6a1799c3bb96aad9 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Thu, 16 Nov 2017 15:54:16 -0600 Subject: [PATCH 22/23] fix output format --- cmd/traffic_cache_tool/CacheTool.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 5bf998bc000..8de3e4a635a 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -44,6 +44,10 @@ #include #include +#define __STDC_FORMAT_MACROS +#include + + using ts::Bytes; using ts::Megabytes; using ts::CacheStoreBlocks; @@ -1291,9 +1295,9 @@ Cache::dumpSpans(SpanDumpDepth depth) if (nullptr == span->_header) { std::cout << "Span: " << span->_path << " is uninitialized" << std::endl; } else { - std::cout << "Span: " << span->_path << " " << span->_header->num_volumes << " Volumes " << span->_header->num_used - << " in use " << span->_header->num_free << " free " << span->_header->num_diskvol_blks << " stripes " - << span->_header->num_blocks.value() << " blocks" << std::endl; + std::cout << "Span: " << span->_path << " #Volumes: " << span->_header->num_volumes <<" #in use: "<< span->_header->num_used + << " #free: " << span->_header->num_free << " #stripes: " << span->_header->num_diskvol_blks << " Len(bytes): " + << span->_header->num_blocks.value() << std::endl; for (auto stripe : span->_stripes) { std::cout << " : " From cf944b2f47c3075f255ba9ebf40acea0cd02f472 Mon Sep 17 00:00:00 2001 From: Persia Aziz Date: Thu, 16 Nov 2017 17:10:14 -0600 Subject: [PATCH 23/23] free list --- cmd/traffic_cache_tool/CacheTool.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/traffic_cache_tool/CacheTool.cc b/cmd/traffic_cache_tool/CacheTool.cc index 8de3e4a635a..a8d2e8f9d30 100644 --- a/cmd/traffic_cache_tool/CacheTool.cc +++ b/cmd/traffic_cache_tool/CacheTool.cc @@ -1899,6 +1899,14 @@ Clear_Span(std::string devicePath) return zret; } +Errata +Check_Freelist(std::string devicePath) +{ + Errata zret; + printf("cache or cash %s\n",devicePath.data()); + return zret; +} + int main(int argc, char *argv[]) { @@ -1937,7 +1945,9 @@ main(int argc, char *argv[]) .subCommand(std::string("stripes"), std::string("List the stripes"), []() { return List_Stripes(Cache::SpanDumpDepth::STRIPE); }); Commands.add(std::string("clear"), std::string("Clear spans"), &Clear_Spans); - Commands.add(std::string("check"), std::string("cache check"), &dir_check); + Commands.add(std::string("dir_check"), std::string("cache check")) + .subCommand(std::string("full"), std::string("Full report of the cache storage"), &dir_check) + .subCommand(std::string("freelist"),std::string("check the freelist for loop"), [&](int, char *argv[]) {return Check_Freelist(inputFile); }); Commands.add(std::string("volumes"), std::string("Volumes"), &Simulate_Span_Allocation); Commands.add(std::string("alloc"), std::string("Storage allocation")) .subCommand(std::string("free"), std::string("Allocate storage on free (empty) spans"), &Cmd_Allocate_Empty_Spans);