From f1dae71722e2ece93a12599a6c640800271938fe Mon Sep 17 00:00:00 2001 From: Griffen Elliott Date: Thu, 12 Mar 2026 23:11:56 +0000 Subject: [PATCH 1/3] Fix tokenize_and_concatenate splitting tokens across chunk boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, text was split into 20 chunks by character count before tokenizing. This could cut words in half at chunk boundaries, producing token pairs that would never occur in naturally tokenized text. Fix splits chunks at whitespace boundaries instead, ensuring no word is ever divided between chunks. Chunk lengths become slightly uneven but this has no effect on correctness — the tokenizer already handles variable-length inputs via padding. Adds regression test that verifies all consecutive token pairs in the output also appear in a clean single-pass tokenization of the same text. Fixes #1133 --- tests/unit/test_utils.py | 46 +++++++++++++++++++++++++++++++++++++++ transformer_lens/utils.py | 14 ++++++++++-- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 242df3987..3766baaeb 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -433,3 +433,49 @@ def test_init_xavier_normal(self, d_model, d_mlp): x_new = nn.Parameter(torch.empty(2, d_model, 137)) utils.init_xavier_normal_(x_new) assert torch.allclose(x_new, x, rtol=1e-2) + +class TestTokenizeAndConcatenate: + """Tests for tokenize_and_concatenate utility function.""" + + def test_no_split_tokens_across_chunks(self): + """ + Regression test for issue #1133. + tokenize_and_concatenate previously split text into chunks by character + count, which could cut words in half and produce token pairs that would + never occur in naturally tokenized text. This test verifies that all + tokens in the output also appear consecutively in a clean tokenization + of the same text, confirming no artificial token pairs were introduced. + """ + from datasets import Dataset + from transformers import AutoTokenizer + + tokenizer = AutoTokenizer.from_pretrained("gpt2") + + # Construct text where character-based splitting would cut mid-word. + # Repeating a long word many times ensures a chunk boundary falls inside it. + text = "Military " * 500 + dataset = Dataset.from_dict({"text": [text]}) + + result = utils.tokenize_and_concatenate( + dataset, + tokenizer, + streaming=False, + max_length=64, + add_bos_token=False, + ) + + # Tokenize the same text cleanly in one shot (no chunking) + clean_tokens = tokenizer(text, return_tensors="np")["input_ids"].flatten() + + # Build a set of all consecutive pairs from the clean tokenization + clean_pairs = set(zip(clean_tokens[:-1], clean_tokens[1:])) + + # Every consecutive pair in our output must exist in the clean pairs + output_tokens = result["tokens"].numpy().flatten() + for i in range(len(output_tokens) - 1): + pair = (output_tokens[i], output_tokens[i + 1]) + assert pair in clean_pairs, ( + f"Token pair {pair} found in tokenize_and_concatenate output " + f"but never occurs in natural tokenization. " + f"This indicates a word was split across chunk boundaries." + ) \ No newline at end of file diff --git a/transformer_lens/utils.py b/transformer_lens/utils.py index d13371cc9..391cf6f8f 100644 --- a/transformer_lens/utils.py +++ b/transformer_lens/utils.py @@ -367,10 +367,20 @@ def tokenize_function(examples: dict[str, list[str]]) -> dict[str, np.ndarray]: if not full_text.strip(): return {"tokens": np.array([], dtype=np.int64)} - # Divide into 20 chunks of ~ equal length + # Divide into 20 chunks of ~ equal length, splitting at whitespace + # boundaries to avoid cutting words in half (which creates token pairs + # that would never occur in naturally tokenized text - see issue #1133) num_chunks = 20 chunk_length = (len(full_text) - 1) // num_chunks + 1 - chunks = [full_text[i * chunk_length : (i + 1) * chunk_length] for i in range(num_chunks)] + chunks = [] + start = 0 + for i in range(num_chunks): + end = min(start + chunk_length, len(full_text)) + # Advance end to the next whitespace boundary to avoid splitting mid-token + while end < len(full_text) and not full_text[end].isspace(): + end += 1 + chunks.append(full_text[start:end]) + start = end # Tokenize the chunks in parallel. Uses NumPy because HuggingFace map doesn't want tensors returned tokens = tokenizer(chunks, return_tensors="np", padding=True)["input_ids"].flatten() # Drop padding tokens From 5ec594c77ebb122693caa3df81997bac5a701e89 Mon Sep 17 00:00:00 2001 From: Griffen Elliott Date: Sat, 14 Mar 2026 10:09:39 +0000 Subject: [PATCH 2/3] Handle no-whitespace edge case in chunk boundary search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a chunk contains no whitespace, the previous fix would advance the boundary search to the end of the string, consuming all remaining text in a single chunk. Fix bounds the lookahead to chunk_length // 10 characters. If no whitespace is found within that window, the cut falls back to the original character boundary — degrading gracefully rather than producing malformed chunks. Addresses feedback from issue #1133. --- transformer_lens/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/transformer_lens/utils.py b/transformer_lens/utils.py index 391cf6f8f..efefe76d0 100644 --- a/transformer_lens/utils.py +++ b/transformer_lens/utils.py @@ -374,10 +374,15 @@ def tokenize_function(examples: dict[str, list[str]]) -> dict[str, np.ndarray]: chunk_length = (len(full_text) - 1) // num_chunks + 1 chunks = [] start = 0 + lookahead = chunk_length // 10 for i in range(num_chunks): end = min(start + chunk_length, len(full_text)) - # Advance end to the next whitespace boundary to avoid splitting mid-token - while end < len(full_text) and not full_text[end].isspace(): + # Advance end to the next whitespace boundary to avoid splitting mid-token. + # Lookahead is bounded so pathological inputs (e.g. no whitespace) degrade + # gracefully to character-based splitting rather than consuming the rest of + # the string. + boundary = min(end + lookahead, len(full_text)) + while end < boundary and not full_text[end].isspace(): end += 1 chunks.append(full_text[start:end]) start = end From dc70868fcf8e2a1c2929d65b9a57cb8bba69514f Mon Sep 17 00:00:00 2001 From: jlarson4 Date: Mon, 16 Mar 2026 09:10:46 -0500 Subject: [PATCH 3/3] Updating Interactive Neuroscope, CI to properly install demo --- .github/workflows/checks.yml | 2 +- demos/Interactive_Neuroscope.ipynb | 82 ++++++------------------------ 2 files changed, 17 insertions(+), 67 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8dd414960..f8f069035 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -174,7 +174,7 @@ jobs: - name: Install dependencies run: | poetry check --lock - poetry install --with dev,jupyter + poetry install --with dev,jupyter,demo - name: Install pandoc uses: awalsh128/cache-apt-pkgs-action@latest with: diff --git a/demos/Interactive_Neuroscope.ipynb b/demos/Interactive_Neuroscope.ipynb index bc288411a..d4bffb6a6 100644 --- a/demos/Interactive_Neuroscope.ipynb +++ b/demos/Interactive_Neuroscope.ipynb @@ -101,18 +101,18 @@ "evalue": "tokenizers>=0.21,<0.22 is required for a normal functioning of this module, but found tokenizers==0.22.2.\nTry: `pip install transformers -U` or `pip install -e '.[dev]'` if you're working with git main", "output_type": "error", "traceback": [ - "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m", - "\u001B[0;31mImportError\u001B[0m Traceback (most recent call last)", - "Cell \u001B[0;32mIn[8], line 3\u001B[0m\n\u001B[1;32m 1\u001B[0m \u001B[38;5;66;03m# NBVAL_IGNORE_OUTPUT\u001B[39;00m\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mgradio\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mgr\u001B[39;00m\n\u001B[0;32m----> 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtransformer_lens\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m HookedTransformer\n\u001B[1;32m 4\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtransformer_lens\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m to_numpy\n\u001B[1;32m 5\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mIPython\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mdisplay\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m HTML\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/transformer_lens/__init__.py:1\u001B[0m\n\u001B[0;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m hook_points\n\u001B[1;32m 2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m utils\n\u001B[1;32m 3\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m evals\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/transformer_lens/hook_points.py:20\u001B[0m\n\u001B[1;32m 17\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mhooks\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mhooks\u001B[39;00m\n\u001B[1;32m 18\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Tensor\n\u001B[0;32m---> 20\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtransformer_lens\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Slice, SliceInput\n\u001B[1;32m 23\u001B[0m \u001B[38;5;129m@dataclass\u001B[39m\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mclass\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mLensHandle\u001B[39;00m:\n\u001B[1;32m 25\u001B[0m \u001B[38;5;250m \u001B[39m\u001B[38;5;124;03m\"\"\"Dataclass that holds information about a PyTorch hook.\"\"\"\u001B[39;00m\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/transformer_lens/utils.py:24\u001B[0m\n\u001B[1;32m 22\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mnn\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mnn\u001B[39;00m\n\u001B[1;32m 23\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mnn\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mfunctional\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mas\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mF\u001B[39;00m\n\u001B[0;32m---> 24\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtransformers\u001B[39;00m\n\u001B[1;32m 25\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mdatasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01marrow_dataset\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Dataset\n\u001B[1;32m 26\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mdatasets\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mload\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m load_dataset\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/__init__.py:27\u001B[0m\n\u001B[1;32m 24\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtyping\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m TYPE_CHECKING\n\u001B[1;32m 26\u001B[0m \u001B[38;5;66;03m# Check the dependencies satisfy the minimal versions required.\u001B[39;00m\n\u001B[0;32m---> 27\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m dependency_versions_check\n\u001B[1;32m 28\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m (\n\u001B[1;32m 29\u001B[0m OptionalDependencyNotAvailable,\n\u001B[1;32m 30\u001B[0m _LazyModule,\n\u001B[0;32m (...)\u001B[0m\n\u001B[1;32m 36\u001B[0m is_pretty_midi_available,\n\u001B[1;32m 37\u001B[0m )\n\u001B[1;32m 39\u001B[0m \u001B[38;5;66;03m# Note: the following symbols are deliberately exported with `as`\u001B[39;00m\n\u001B[1;32m 40\u001B[0m \u001B[38;5;66;03m# so that mypy, pylint or other static linters can recognize them,\u001B[39;00m\n\u001B[1;32m 41\u001B[0m \u001B[38;5;66;03m# given that they are not exported using `__all__` in this file.\u001B[39;00m\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/dependency_versions_check.py:57\u001B[0m\n\u001B[1;32m 54\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m is_accelerate_available():\n\u001B[1;32m 55\u001B[0m \u001B[38;5;28;01mcontinue\u001B[39;00m \u001B[38;5;66;03m# not required, check version only if installed\u001B[39;00m\n\u001B[0;32m---> 57\u001B[0m \u001B[43mrequire_version_core\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdeps\u001B[49m\u001B[43m[\u001B[49m\u001B[43mpkg\u001B[49m\u001B[43m]\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m 58\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m 59\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mcan\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mt find \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mpkg\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m in \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mdeps\u001B[38;5;241m.\u001B[39mkeys()\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m, check dependency_versions_table.py\u001B[39m\u001B[38;5;124m\"\u001B[39m)\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/utils/versions.py:117\u001B[0m, in \u001B[0;36mrequire_version_core\u001B[0;34m(requirement)\u001B[0m\n\u001B[1;32m 115\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"require_version wrapper which emits a core-specific hint on failure\"\"\"\u001B[39;00m\n\u001B[1;32m 116\u001B[0m hint \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mTry: `pip install transformers -U` or `pip install -e \u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m.[dev]\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m` if you\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mre working with git main\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[0;32m--> 117\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mrequire_version\u001B[49m\u001B[43m(\u001B[49m\u001B[43mrequirement\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mhint\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/utils/versions.py:111\u001B[0m, in \u001B[0;36mrequire_version\u001B[0;34m(requirement, hint)\u001B[0m\n\u001B[1;32m 109\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m want_ver \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m 110\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m op, want_ver \u001B[38;5;129;01min\u001B[39;00m wanted\u001B[38;5;241m.\u001B[39mitems():\n\u001B[0;32m--> 111\u001B[0m \u001B[43m_compare_versions\u001B[49m\u001B[43m(\u001B[49m\u001B[43mop\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mgot_ver\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mwant_ver\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mrequirement\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mpkg\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mhint\u001B[49m\u001B[43m)\u001B[49m\n", - "File \u001B[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/utils/versions.py:44\u001B[0m, in \u001B[0;36m_compare_versions\u001B[0;34m(op, got_ver, want_ver, requirement, pkg, hint)\u001B[0m\n\u001B[1;32m 39\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mValueError\u001B[39;00m(\n\u001B[1;32m 40\u001B[0m \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mUnable to compare versions for \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mrequirement\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m: need=\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mwant_ver\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m found=\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mgot_ver\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m. This is unusual. Consider\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 41\u001B[0m \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m reinstalling \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mpkg\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m.\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 42\u001B[0m )\n\u001B[1;32m 43\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m ops[op](version\u001B[38;5;241m.\u001B[39mparse(got_ver), version\u001B[38;5;241m.\u001B[39mparse(want_ver)):\n\u001B[0;32m---> 44\u001B[0m \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mImportError\u001B[39;00m(\n\u001B[1;32m 45\u001B[0m \u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mrequirement\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m is required for a normal functioning of this module, but found \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mpkg\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m==\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mgot_ver\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m.\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mhint\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m 46\u001B[0m )\n", - "\u001B[0;31mImportError\u001B[0m: tokenizers>=0.21,<0.22 is required for a normal functioning of this module, but found tokenizers==0.22.2.\nTry: `pip install transformers -U` or `pip install -e '.[dev]'` if you're working with git main" + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# NBVAL_IGNORE_OUTPUT\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mgradio\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mgr\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtransformer_lens\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m HookedTransformer\n\u001b[1;32m 4\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtransformer_lens\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m to_numpy\n\u001b[1;32m 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mIPython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdisplay\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m HTML\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/transformer_lens/__init__.py:1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m hook_points\n\u001b[1;32m 2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m utils\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m evals\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/transformer_lens/hook_points.py:20\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mhooks\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mhooks\u001b[39;00m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Tensor\n\u001b[0;32m---> 20\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtransformer_lens\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Slice, SliceInput\n\u001b[1;32m 23\u001b[0m \u001b[38;5;129m@dataclass\u001b[39m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mLensHandle\u001b[39;00m:\n\u001b[1;32m 25\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Dataclass that holds information about a PyTorch hook.\"\"\"\u001b[39;00m\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/transformer_lens/utils.py:24\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnn\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mnn\u001b[39;00m\n\u001b[1;32m 23\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtorch\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mnn\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mfunctional\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mF\u001b[39;00m\n\u001b[0;32m---> 24\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtransformers\u001b[39;00m\n\u001b[1;32m 25\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mdatasets\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01marrow_dataset\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m Dataset\n\u001b[1;32m 26\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mdatasets\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mload\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m load_dataset\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/__init__.py:27\u001b[0m\n\u001b[1;32m 24\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtyping\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m TYPE_CHECKING\n\u001b[1;32m 26\u001b[0m \u001b[38;5;66;03m# Check the dependencies satisfy the minimal versions required.\u001b[39;00m\n\u001b[0;32m---> 27\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m dependency_versions_check\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m (\n\u001b[1;32m 29\u001b[0m OptionalDependencyNotAvailable,\n\u001b[1;32m 30\u001b[0m _LazyModule,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 36\u001b[0m is_pretty_midi_available,\n\u001b[1;32m 37\u001b[0m )\n\u001b[1;32m 39\u001b[0m \u001b[38;5;66;03m# Note: the following symbols are deliberately exported with `as`\u001b[39;00m\n\u001b[1;32m 40\u001b[0m \u001b[38;5;66;03m# so that mypy, pylint or other static linters can recognize them,\u001b[39;00m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;66;03m# given that they are not exported using `__all__` in this file.\u001b[39;00m\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/dependency_versions_check.py:57\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_accelerate_available():\n\u001b[1;32m 55\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m \u001b[38;5;66;03m# not required, check version only if installed\u001b[39;00m\n\u001b[0;32m---> 57\u001b[0m \u001b[43mrequire_version_core\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdeps\u001b[49m\u001b[43m[\u001b[49m\u001b[43mpkg\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 59\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcan\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt find \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpkg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m in \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mdeps\u001b[38;5;241m.\u001b[39mkeys()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, check dependency_versions_table.py\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/utils/versions.py:117\u001b[0m, in \u001b[0;36mrequire_version_core\u001b[0;34m(requirement)\u001b[0m\n\u001b[1;32m 115\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"require_version wrapper which emits a core-specific hint on failure\"\"\"\u001b[39;00m\n\u001b[1;32m 116\u001b[0m hint \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTry: `pip install transformers -U` or `pip install -e \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.[dev]\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m` if you\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mre working with git main\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 117\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequire_version\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequirement\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhint\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/utils/versions.py:111\u001b[0m, in \u001b[0;36mrequire_version\u001b[0;34m(requirement, hint)\u001b[0m\n\u001b[1;32m 109\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m want_ver \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 110\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m op, want_ver \u001b[38;5;129;01min\u001b[39;00m wanted\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 111\u001b[0m \u001b[43m_compare_versions\u001b[49m\u001b[43m(\u001b[49m\u001b[43mop\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgot_ver\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwant_ver\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mrequirement\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpkg\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhint\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Projects/_TRANSFORMER_LENS/TransformerLens/.venv/lib/python3.12/site-packages/transformers/utils/versions.py:44\u001b[0m, in \u001b[0;36m_compare_versions\u001b[0;34m(op, got_ver, want_ver, requirement, pkg, hint)\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m 40\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnable to compare versions for \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrequirement\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: need=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mwant_ver\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m found=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mgot_ver\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. This is unusual. Consider\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 41\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m reinstalling \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpkg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 42\u001b[0m )\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m ops[op](version\u001b[38;5;241m.\u001b[39mparse(got_ver), version\u001b[38;5;241m.\u001b[39mparse(want_ver)):\n\u001b[0;32m---> 44\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[1;32m 45\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mrequirement\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m is required for a normal functioning of this module, but found \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpkg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m==\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mgot_ver\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mhint\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 46\u001b[0m )\n", + "\u001b[0;31mImportError\u001b[0m: tokenizers>=0.21,<0.22 is required for a normal functioning of this module, but found tokenizers==0.22.2.\nTry: `pip install transformers -U` or `pip install -e '.[dev]'` if you're working with git main" ] } ], @@ -417,60 +417,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running on local URL: http://127.0.0.1:7860\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Running on public URL: https://7a615281b36111d2e4.gradio.live\n", - "\n", - "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n" - ] - }, - { - "data": { - "text/html": [ - "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# NBVAL_IGNORE_OUTPUT\n", - "demo.launch(share=True, height=1000)" - ] + "outputs": [], + "source": "# NBVAL_SKIP\ndemo.launch(share=True, height=1000)" } ], "metadata": { @@ -500,4 +450,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file