diff --git a/notebooks/bug_in_the_code_stack_experiment_litellm_deepseek_chat25.ipynb b/notebooks/bug_in_the_code_stack_experiment_litellm_deepseek_chat25.ipynb new file mode 100644 index 0000000..a25767a --- /dev/null +++ b/notebooks/bug_in_the_code_stack_experiment_litellm_deepseek_chat25.ipynb @@ -0,0 +1 @@ +{"cells":[{"cell_type":"markdown","metadata":{"id":"9S108Nhhpho_"},"source":["# Bug In The Code Stack - Experiment w/t LiteLLM (Meta: Llama 3)"]},{"cell_type":"markdown","metadata":{"id":"enjtXsLX8Va1"},"source":["# `.env`\n","\n","- Create a `.env` file with the following info:\n","\n","```\n","GROQ_API_KEY=...\n","```"]},{"cell_type":"code","execution_count":14,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":330},"executionInfo":{"elapsed":2872,"status":"ok","timestamp":1714921639040,"user":{"displayName":"Andy Lee","userId":"09300480258258053500"},"user_tz":240},"id":"rvky8ng9p04O","outputId":"c583a0f0-a00b-4377-f56d-62d5129726c0"},"outputs":[{"data":{"text/html":["
| \n"," | output | \n","output_missing_colon | \n","bug_line_number_missing_colon | \n","output_missing_parenthesis | \n","bug_line_number_missing_parenthesis | \n","output_missing_quotation | \n","bug_line_number_missing_quotation | \n","output_missing_comma | \n","bug_line_number_missing_comma | \n","output_mismatched_quotation | \n","bug_line_number_mismatched_quotation | \n","output_mismatched_bracket | \n","bug_line_number_mismatched_bracket | \n","output_keywords_as_identifier | \n","bug_line_number_keywords_as_identifier | \n","
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n","# Python code\\ndef sum_sequence(sequence):\\n ... | \n","# Python code\\ndef sum_sequence(sequence)\\n s... | \n","2 | \n","# Python code\\ndef sum_sequence(sequence:\\n s... | \n","2 | \n","\n"," | -1 | \n","\n"," | -1 | \n","\n"," | -1 | \n","# Python code\\ndef sum_sequence(sequence]:\\n ... | \n","2 | \n","# Python code\\ndef while(sequence):\\n sum = 0... | \n","2 | \n","
| 1 | \n","[x*x for x in [1, 2, 3, 5, 8, 13]] | \n","\n"," | -1 | \n","\n"," | -1 | \n","\n"," | -1 | \n","[x*x for x in [1 2, 3, 5, 8, 13]] | \n","1 | \n","\n"," | -1 | \n","[x*x for x in [1, 2, 3, 5, 8, 13}] | \n","1 | \n","[for*x for x in [1, 2, 3, 5, 8, 13]] | \n","1 | \n","
| 2 | \n","def is_palindrome(s):\\n '''This function re... | \n","def is_palindrome(s)\\n '''This function ret... | \n","1 | \n","def is_palindrome(s:\\n '''This function ret... | \n","1 | \n","def is_palindrome(s):\\n ''This function ret... | \n","2 | \n","\n"," | -1 | \n","def is_palindrome(s):\\n '\"'This function re... | \n","2 | \n","def is_palindrome(s]:\\n '''This function re... | \n","1 | \n","def if(s):\\n '''This function returns True ... | \n","1 | \n","
| 3 | \n","def double_add(num1, num2): \\n return (num1... | \n","\n"," | -1 | \n","def double_add(num1, num2: \\n return (num1 ... | \n","1 | \n","\n"," | -1 | \n","def double_add(num1 num2): \\n return (num1 ... | \n","1 | \n","\n"," | -1 | \n","def double_add(num1, num2]: \\n return (num1... | \n","1 | \n","def is(num1, num2): \\n return (num1 + num2)... | \n","1 | \n","
| 4 | \n","def find_numbers_divisible_by(m, n, divisor):\\... | \n","def find_numbers_divisible_by(m, n, divisor)\\n... | \n","1 | \n","def find_numbers_divisible_by(m, n, divisor:\\n... | \n","1 | \n","\n"," | -1 | \n","def find_numbers_divisible_by(m n, divisor):\\n... | \n","1 | \n","\n"," | -1 | \n","def find_numbers_divisible_by(m, n, divisor}:\\... | \n","1 | \n","def not(m, n, divisor):\\n divisible_numbers... | \n","1 | \n","
| \n"," | output | \n","output_missing_colon | \n","bug_line_number_missing_colon | \n","output_missing_parenthesis | \n","bug_line_number_missing_parenthesis | \n","output_missing_quotation | \n","bug_line_number_missing_quotation | \n","output_missing_comma | \n","bug_line_number_missing_comma | \n","output_mismatched_quotation | \n","bug_line_number_mismatched_quotation | \n","output_mismatched_bracket | \n","bug_line_number_mismatched_bracket | \n","output_keywords_as_identifier | \n","bug_line_number_keywords_as_identifier | \n","
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n","# Python code\\ndef sum_sequence(sequence):\\n ... | \n","# Python code\\ndef sum_sequence(sequence)\\n s... | \n","2 | \n","# Python code\\ndef sum_sequence(sequence:\\n s... | \n","2 | \n","\n"," | -1 | \n","\n"," | -1 | \n","\n"," | -1 | \n","# Python code\\ndef sum_sequence(sequence]:\\n ... | \n","2 | \n","# Python code\\ndef while(sequence):\\n sum = 0... | \n","2 | \n","
| 1 | \n","[x*x for x in [1, 2, 3, 5, 8, 13]] | \n","\n"," | -1 | \n","\n"," | -1 | \n","\n"," | -1 | \n","[x*x for x in [1 2, 3, 5, 8, 13]] | \n","1 | \n","\n"," | -1 | \n","[x*x for x in [1, 2, 3, 5, 8, 13}] | \n","1 | \n","[for*x for x in [1, 2, 3, 5, 8, 13]] | \n","1 | \n","
| 2 | \n","def is_palindrome(s):\\n '''This function re... | \n","def is_palindrome(s)\\n '''This function ret... | \n","1 | \n","def is_palindrome(s:\\n '''This function ret... | \n","1 | \n","def is_palindrome(s):\\n ''This function ret... | \n","2 | \n","\n"," | -1 | \n","def is_palindrome(s):\\n '\"'This function re... | \n","2 | \n","def is_palindrome(s]:\\n '''This function re... | \n","1 | \n","def if(s):\\n '''This function returns True ... | \n","1 | \n","
| 3 | \n","def double_add(num1, num2): \\n return (num1... | \n","\n"," | -1 | \n","def double_add(num1, num2: \\n return (num1 ... | \n","1 | \n","\n"," | -1 | \n","def double_add(num1 num2): \\n return (num1 ... | \n","1 | \n","\n"," | -1 | \n","def double_add(num1, num2]: \\n return (num1... | \n","1 | \n","def is(num1, num2): \\n return (num1 + num2)... | \n","1 | \n","
| 4 | \n","def find_numbers_divisible_by(m, n, divisor):\\... | \n","def find_numbers_divisible_by(m, n, divisor)\\n... | \n","1 | \n","def find_numbers_divisible_by(m, n, divisor:\\n... | \n","1 | \n","\n"," | -1 | \n","def find_numbers_divisible_by(m n, divisor):\\n... | \n","1 | \n","\n"," | -1 | \n","def find_numbers_divisible_by(m, n, divisor}:\\... | \n","1 | \n","def not(m, n, divisor):\\n divisible_numbers... | \n","1 | \n","
| \n"," | target_length | \n","target_depth | \n","iter_no | \n","bug_line_number | \n","bug_type | \n","pred_bug_line_number | \n","pred_bug_type | \n","result | \n","
|---|---|---|---|---|---|---|---|---|
| 0 | \n","500 | \n","0.0 | \n","0 | \n","3 | \n","missing_colon | \n","3 | \n","missing_colon | \n","True | \n","
| 1 | \n","500 | \n","0.0 | \n","1 | \n","1 | \n","missing_comma | \n","1 | \n","missing_comma | \n","True | \n","
| 2 | \n","500 | \n","0.0 | \n","2 | \n","1 | \n","missing_colon | \n","1 | \n","missing_colon | \n","True | \n","
| 3 | \n","500 | \n","0.0 | \n","3 | \n","2 | \n","missing_colon | \n","2 | \n","missing_colon | \n","True | \n","
| 4 | \n","500 | \n","0.0 | \n","4 | \n","1 | \n","keywords_as_identifier | \n","2 | \n","keywords_as_identifier | \n","False | \n","