Update tracked sheet when pushing

cogs/diff.py

@@ -4,7 +4,13 @@
 import tabulate
 
 from cogs.exceptions import DiffError
-from cogs.helpers import get_diff, get_tracked_sheets, set_logging, validate_cogs_project
+from cogs.helpers import (
+    get_cached_path,
+    get_diff,
+    get_tracked_sheets,
+    set_logging,
+    validate_cogs_project,
+)
 
 
 def close_screen(stdscr):
@@ -216,11 +222,11 @@ def diff(paths=None, use_screen=True, verbose=False):
 
     diffs = {}
     for sheet_title, details in sheets.items():
-        path_name = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower())
-        remote = f"{cogs_dir}/tracked/{path_name}.tsv"
+        cached = get_cached_path(cogs_dir, sheet_title)
         local = details["Path"]
-        if os.path.exists(local) and os.path.exists(remote):
-            sheet_diff = get_diff(local, remote)
+        if os.path.exists(local) and os.path.exists(cached):
+            # Consider remote (cached) the old version to diff off of
+            sheet_diff = get_diff(cached, local)
             diffs[sheet_title] = sheet_diff
 
     if not diffs:

cogs/fetch.py

@@ -9,6 +9,7 @@
 import gspread_formatting as gf
 
 from cogs.helpers import (
+    get_cached_path,
     get_credentials,
     get_config,
     get_format_dict,
@@ -412,8 +413,8 @@ def fetch(verbose=False):
             sheet_notes[st] = cell_to_note
 
         # Write values to .cogs/tracked/{sheet title}.tsv
-        sheet_path = re.sub(r"[^A-Za-z0-9]+", "_", st.lower()).strip("_")
-        with open(f"{cogs_dir}/tracked/{sheet_path}.tsv", "w") as f:
+        cached_path = get_cached_path(cogs_dir, st)
+        with open(cached_path, "w") as f:
             lines = sheet.get_all_values()
             writer = csv.writer(f, delimiter="\t", lineterminator="\n")
             writer.writerows(lines)

cogs/helpers.py

@@ -26,6 +26,12 @@
 credential_keys = []
 
 
+def get_cached_path(cogs_dir, sheet_title):
+    """Return the path to the cached version of a sheet based on its title."""
+    filename = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower())
+    return f"{cogs_dir}/tracked/{filename}.tsv"
+
+
 def get_cached_sheets(cogs_dir):
     """Return a list of names of cached sheets from .cogs/tracked. These are any sheets that have
     been downloaded from the remote spreadsheet into the .cogs directory as TSVs. They may or may
@@ -130,19 +136,18 @@ def get_data_validation(cogs_dir):
     return sheet_to_dv_rules
 
 
-def get_diff(local, remote):
-    """Return the diff between a local and remote sheet as a list of lines (list of cell values)
-    with daff 'highlighter' formatting. The 'highlight' is appended to the beginning of the line as:
+def get_diff(left, right):
+    """Return the diff between a left (old) and right (new) sheet as a list of lines (list of cell
+    values) with daff 'highlighter' formatting. The 'highlight' is appended to the beginning of the
+    line as:
     - '+++' for added lines
     - '->' for changed lines
     - '...' for omitted rows
     - '---' for removed lines
-    - '' for unchanged lines
-    The remote table is the 'old' version and the local table is the 'new' version."""
-    local_data = []
-    with open(local, "r") as f:
-        # Local might be CSV or TSV
-        if local.endswith("csv"):
+    - '' for unchanged lines"""
+    left_data = []
+    with open(left, "r") as f:
+        if left.endswith("csv"):
             reader = csv.reader(f)
         else:
             reader = csv.reader(f, delimiter="\t")
@@ -152,36 +157,38 @@ def get_diff(local, remote):
             # No data
             header = None
         if header:
-            local_data.append(header)
+            left_data.append(header)
             for row in reader:
                 if len(row) < len(header):
                     add = [""] * (len(header) - len(row))
                     row.extend(add)
-                local_data.append(row)
+                left_data.append(row)
 
-    remote_data = []
-    with open(remote, "r") as f:
-        # Remote is always TSV
-        reader = csv.reader(f, delimiter="\t")
+    right_data = []
+    with open(right, "r") as f:
+        if right.endswith("csv"):
+            reader = csv.reader(f)
+        else:
+            reader = csv.reader(f, delimiter="\t")
         try:
             header = next(reader)
         except StopIteration:
             # No data
             header = None
         if header:
-            remote_data.append(header)
+            right_data.append(header)
             for row in reader:
                 if len(row) < len(header):
                     add = [""] * (len(header) - len(row))
                     row.extend(add)
-                remote_data.append(row)
+                right_data.append(row)
 
-    if not local_data and not remote_data:
+    if not right_data and not left_data:
         return []
 
-    local_table = PythonTableView(local_data)
-    remote_table = PythonTableView(remote_data)
-    align = Coopy.compareTables(remote_table, local_table).align()
+    right_table = PythonTableView(right_data)
+    left_table = PythonTableView(left_data)
+    align = Coopy.compareTables(left_table, right_table).align()
 
     data_diff = []
     table_diff = PythonTableView(data_diff)

cogs/pull.py

@@ -5,6 +5,7 @@
 import shutil
 
 from cogs.helpers import (
+    get_cached_path,
     get_cached_sheets,
     get_renamed_sheets,
     get_tracked_sheets,
@@ -32,15 +33,14 @@ def pull(verbose=False):
     remove_sheets = [s for s in cached_sheets if s not in tracked_cached]
 
     for sheet_title, details in tracked_sheets.items():
-        path_name = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower())
-        cached_sheet = f"{cogs_dir}/tracked/{path_name}.tsv"
+        cached_path = get_cached_path(cogs_dir, sheet_title)
         local_sheet = details["Path"]
-        if os.path.exists(cached_sheet):
+        if os.path.exists(cached_path):
             logging.info(f"Writing '{sheet_title}' to {local_sheet}")
             if local_sheet.endswith(".csv"):
-                copy_to_csv(cached_sheet, local_sheet)
+                copy_to_csv(cached_path, local_sheet)
             else:
-                shutil.copyfile(cached_sheet, local_sheet)
+                shutil.copyfile(cached_path, local_sheet)
     for sheet_title in remove_sheets:
         logging.info(f"Removing '{sheet_title}' from cached sheets")
         os.remove(f"{cogs_dir}/tracked/{sheet_title}.tsv")

cogs/push.py

@@ -10,6 +10,7 @@
     get_tracked_sheets,
     set_logging,
     validate_cogs_project,
+    get_cached_path,
     get_config,
     get_client_from_config,
     get_renamed_sheets,
@@ -40,7 +41,8 @@ def clear_remote_sheets(spreadsheet, renamed_local):
 
 
 def push_data(cogs_dir, spreadsheet, tracked_sheets, remote_sheets):
-    """Push all tracked sheets to the spreadsheet. Return updated rows for sheet.tsv."""
+    """Push all tracked sheets to the spreadsheet. Update sheets in COGS tracked directory. Return
+    updated rows for sheet.tsv."""
     sheet_rows = []
     for sheet_title, details in tracked_sheets.items():
         sheet_path = details["Path"]
@@ -52,13 +54,17 @@ def push_data(cogs_dir, spreadsheet, tracked_sheets, remote_sheets):
         if not os.path.exists(sheet_path):
             logging.warning(f"'{sheet_title}' exists remotely but has not been pulled")
             continue
-        with open(sheet_path, "r") as f:
-            reader = csv.reader(f, delimiter=delimiter)
-            for row in reader:
-                row_len = len(row)
-                if row_len > cols:
-                    cols = row_len
-                rows.append(row)
+        with open(sheet_path, "r") as fr:
+            reader = csv.reader(fr, delimiter=delimiter)
+            tracked_sheet = get_cached_path(cogs_dir, sheet_title)
+            with open(tracked_sheet, "w") as fw:
+                writer = csv.writer(fw, delimiter="\t", lineterminator="\n")
+                for row in reader:
+                    writer.writerow(row)
+                    row_len = len(row)
+                    if row_len > cols:
+                        cols = row_len
+                    rows.append(row)
 
         # Set sheet size
         if len(rows) < 500:
@@ -93,8 +99,8 @@ def push_data(cogs_dir, spreadsheet, tracked_sheets, remote_sheets):
         sheet.freeze(frozen_row, frozen_col)
 
         # Copy this table into COGS data
-        path_name = re.sub(r"[^A-Za-z0-9]+", "_", sheet_title.lower())
-        with open(f"{cogs_dir}/tracked/{path_name}.tsv", "w") as f:
+        cached_name = get_cached_path(cogs_dir, sheet_title)
+        with open(cached_name, "w") as f:
             writer = csv.writer(f, delimiter="\t", lineterminator="\n")
             writer.writerows(rows)
     return sheet_rows
@@ -127,28 +133,40 @@ def push_data_validation(spreadsheet, data_validation, tracked_sheets):
             show_ui = False
             if condition.endswith("LIST"):
                 show_ui = True
-            requests.append({"updateCells": {
-                "range": {
-                    "sheetId": sheet_id,
-                    "startRowIndex": start_row - 1,
-                    "endRowIndex": end_row,
-                    "startColumnIndex": start_col - 1,
-                    "endColumnIndex": end_col,
-                },
-                "rows": [
-                    {"values":
-                         {"dataValidation":
-                              {"condition": {"type": condition, "values": value_obj},
-                               "showCustomUi": show_ui}}}],
-                "fields": "dataValidation",
-            }})
+            requests.append(
+                {
+                    "updateCells": {
+                        "range": {
+                            "sheetId": sheet_id,
+                            "startRowIndex": start_row - 1,
+                            "endRowIndex": end_row,
+                            "startColumnIndex": start_col - 1,
+                            "endColumnIndex": end_col,
+                        },
+                        "rows": [
+                            {
+                                "values": {
+                                    "dataValidation": {
+                                        "condition": {"type": condition, "values": value_obj},
+                                        "showCustomUi": show_ui,
+                                    }
+                                }
+                            }
+                        ],
+                        "fields": "dataValidation",
+                    }
+                }
+            )
     if not requests:
         return
     try:
         logging.info(f"adding {len(requests)} data validation rules to spreadsheet")
         spreadsheet.batch_update({"requests": requests})
     except gspread.exceptions.APIError as e:
-        logging.error(f"Unable to add {len(requests)} data validation rules to spreadsheet\n" + e.response.text)
+        logging.error(
+            f"Unable to add {len(requests)} data validation rules to spreadsheet\n"
+            + e.response.text
+        )
 
 
 def push_formats(spreadsheet, id_to_format, sheet_formats):

cogs/rm.py

@@ -1,7 +1,9 @@
 import csv
+import os
 
 from cogs.exceptions import RmError
 from cogs.helpers import (
+    get_cached_path,
     get_tracked_sheets,
     set_logging,
     validate_cogs_project,
@@ -36,11 +38,11 @@ def rm(paths, verbose=False):
             "the spreadsheet must have at least one sheet."
         )
 
-    # Make sure the titles are valid
+    # Remove the cached copies
     for sheet_title in sheets_to_remove.keys():
-        if "." in sheet_title or "/" in sheet_title:
-            # We should probably use a proper way to make sure the file name is in .cogs
-            raise RmError("Invalid title for sheet, cannot contain . or /")
+        cached_path = get_cached_path(cogs_dir, sheet_title)
+        if os.path.exists(cached_path):
+            os.remove(cached_path)
 
     # Update sheet.tsv
     with open(f"{cogs_dir}/sheet.tsv", "w") as f:

cogs/status.py

@@ -108,10 +108,12 @@ def get_changes(cogs_dir, tracked_sheets, renamed):
             local_mod = os.path.getmtime(local_path)
             remote_mod = os.path.getmtime(remote_path)
             if remote_mod > local_mod:
-                diff = get_diff(remote_path, local_path)
+                # Remote is newer
+                diff = get_diff(local_path, remote_path)
                 new_version = "remote"
             else:
-                diff = get_diff(local_path, remote_path)
+                # Local is newer
+                diff = get_diff(remote_path, local_path)
                 new_version = "local"
 
             if len(diff) > 1: