Add CI workflows, linting configuration, and code quality improvements

.github/workflows/race-detection.yml

@@ -0,0 +1,35 @@
+name: Race Detection
+
+on:
+  push:
+    branches: [ main, page-index, page-index-fixes ]
+  pull_request:
+    branches: [ main ]
+
+# Temporarily skip race detection workflows
+
+jobs:
+  race:
+    name: Race Detection
+    runs-on: ubuntu-latest
+    if: false  # Skip race detection for now
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.24'
+
+      - name: Get dependencies
+        run: go mod download
+
+      - name: Run tests with race detector
+        run: go test -race -v ./...
+        env:
+          GORACE: "halt_on_error=1"
+
+      - name: Run benchmarks with race detector
+        run: go test -race -run=^$ -bench=. -benchtime=10x ./...
+        env:
+          GORACE: "halt_on_error=1"

.github/workflows/test-and-lint.yml

@@ -0,0 +1,88 @@
+name: Test and Lint
+
+on:
+  push:
+    branches: [ main, page-index, page-index-fixes ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go-version: ['1.22', '1.23', '1.24']
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Get dependencies
+        run: go mod download
+
+      - name: Run tests
+        run: go test -v -coverprofile=coverage.out ./...
+
+      - name: Upload coverage reports
+        uses: codecov/codecov-action@v3
+        if: matrix.go-version == '1.24'
+        with:
+          file: ./coverage.out
+          fail_ci_if_error: false
+
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.24'
+
+      - name: Run go fmt
+        run: |
+          fmt_output=$(go fmt ./...)
+          if [ -n "$fmt_output" ]; then
+            echo "The following files need formatting:"
+            echo "$fmt_output"
+            exit 1
+          fi
+
+      - name: Run go vet
+        run: go vet ./...
+
+      - name: Install staticcheck
+        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+
+      - name: Run staticcheck
+        run: staticcheck ./...
+
+      - name: Install revive
+        run: go install github.com/mgechev/revive@latest
+
+      - name: Run revive
+        run: revive -config .revive.toml ./...
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs: [test, lint]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.24'
+
+      - name: Build
+        run: go build -v ./...
+
+      - name: Build CLI
+        run: go build -v -o ltx ./cmd/ltx

.gitignore

@@ -1,2 +1,6 @@
 .vscode
 
+# Go binaries
+cmd/ltx/ltx
+ltx
+

.revive.toml

@@ -0,0 +1,56 @@
+ignoreGeneratedHeader = false
+severity = "warning"
+confidence = 0.8
+errorCode = 1
+warningCode = 0
+
+[rule.blank-imports]
+[rule.context-as-argument]
+[rule.context-keys-type]
+[rule.dot-imports]
+[rule.error-return]
+[rule.error-strings]
+[rule.error-naming]
+[rule.exported]
+[rule.if-return]
+[rule.increment-decrement]
+[rule.var-naming]
+[rule.var-declaration]
+[rule.package-comments]
+[rule.range]
+[rule.receiver-naming]
+[rule.time-naming]
+[rule.unexported-return]
+[rule.indent-error-flow]
+[rule.errorf]
+[rule.empty-block]
+[rule.superfluous-else]
+[rule.unused-parameter]
+[rule.unreachable-code]
+[rule.redefines-builtin-id]
+
+# Additional useful rules
+# Disabled cyclomatic and cognitive complexity
+# [rule.cognitive-complexity]
+#   arguments = [15]
+# [rule.cyclomatic]
+#   arguments = [10]
+
+# Disabled line length limit
+# [rule.line-length-limit]
+#   arguments = [120]
+
+[rule.function-result-limit]
+  arguments = [3]
+
+[rule.argument-limit]
+  arguments = [5]
+
+[rule.unnecessary-stmt]
+[rule.deep-exit]
+[rule.duplicated-imports]
+[rule.import-shadowing]
+[rule.bare-return]
+[rule.unused-receiver]
+[rule.unhandled-error]
+  arguments = ["fmt.Printf", "fmt.Println", "fmt.Print", "fmt.Fprintf"]

README.md

@@ -4,7 +4,8 @@ Lite Transaction File (LTX)
 The LTX file format provides a way to store SQLite transactional data in
 a way that can be encrypted and compacted and is optimized for performance.
 
-## File Format
+File Format
+-----------
 
 An LTX file is composed of several sections:
 
@@ -15,8 +16,8 @@ An LTX file is composed of several sections:
 The header contains metadata about the file, the page block contains page
 frames, and the trailer contains checksums of the file and the database end state.
 
-
-#### Header
+Header
+------
 
 The header provides information about the number of page frames as well as
 database information such as the page size and database size. LTX files
@@ -25,21 +26,32 @@ that it represents. A timestamp provides users with a rough approximation of
 the time the transaction occurred and the checksum provides a basic integrity
 check.
 
-| Offset | Size | Description                             |
-| -------| ---- | --------------------------------------- |
-| 0      | 4    | Magic number. Always "LTX1".            |
-| 4      | 4    | Flags. Reserved. Always 0.              |
-| 8      | 4    | Page size, in bytes.                    |
-| 12     | 4    | Size of DB after transaction, in pages. |
-| 16     | 4    | Database ID.                            |
-| 20     | 8    | Minimum transaction ID.                 |
-| 28     | 8    | Maximum transaction ID.                 |
-| 36     | 8    | Timestamp (Milliseconds since epoch)    |
-| 44     | 8    | Pre-apply DB checksum (CRC-ISO-64)      |
-| 52     | 48   | Reserved.                               |
-
-
-#### Page block
+| Offset | Size | Description                                     |
+| -------| ---- | ----------------------------------------------- |
+| 0      | 4    | Magic number. Always "LTX1".                    |
+| 4      | 4    | Flags. See below.                               |
+| 8      | 4    | Page size, in bytes.                            |
+| 12     | 4    | Size of DB after transaction, in pages.         |
+| 16     | 8    | Minimum transaction ID.                         |
+| 24     | 8    | Maximum transaction ID.                         |
+| 32     | 8    | Timestamp (Milliseconds since epoch)            |
+| 40     | 8    | Pre-apply DB checksum (CRC-ISO-64)              |
+| 48     | 8    | File offset in WAL, zero if journal             |
+| 56     | 8    | Size of WAL segment, zero if journal            |
+| 64     | 4    | Salt-1 from WAL, zero if journal or compacted   |
+| 68     | 4    | Salt-2 from WAL, zero if journal or compacted   |
+| 72     | 8    | ID of the node that created file, zero if unset |
+| 80     | 20   | Reserved.                                       |
+
+Header flags
+------------
+
+| Flag       | Description                 |
+| ---------- | --------------------------- |
+| 0x00000001 | Data is compressed with LZ4 |
+
+Page block
+----------
 
 This block stores a series of page headers and page data.
 
@@ -48,8 +60,8 @@ This block stores a series of page headers and page data.
 | 0      | 4    | Page number.                |
 | 4      | N    | Page data.                  |
 
-
-#### Trailer
+Trailer
+-------
 
 The trailer provides checksum for the LTX file data, a rolling checksum of the
 database state after the LTX file is applied, and the checksum of the trailer
@@ -60,4 +72,32 @@ itself.
 | 0      | 8    | Post-apply DB checksum (CRC-ISO-64)     |
 | 8      | 8    | File checksum (CRC-ISO-64)              |
 
+Checksum Design
+---------------
+
+LTX uses checksums in two distinct ways:
+
+Database Checksum
+-----------------
+
+- **Purpose**: Tracks the overall state of the database
+- **Computation**: XOR of all page-level checksums in the database
+- **Maintenance**: Incrementally maintained by removing old page checksums
+  and adding new ones
+- **Storage**: `PreApplyChecksum` and `PostApplyChecksum` fields in header
+  and trailer
+
+File Checksum
+-------------
+
+- **Purpose**: Ensures the LTX file itself hasn't been tampered with
+- **Computation**: Computed over the file contents up to (but not including)
+  the file checksum field in the trailer
+- **Important**: The page index **is included** in the file checksum calculation
+- **Rationale**: Including the page index prevents tampering with page offset/size
+  mappings, which could redirect reads to incorrect data
+- **Storage**: `FileChecksum` field in the trailer
 
+**Security**: The page index is included in the file checksum to detect tampering
+with page mappings. While page data itself has individual checksums, the index
+mappings must also be protected to prevent malicious redirection attacks.

checksum.go

@@ -148,11 +148,13 @@ func (c Checksum) String() string {
 	return fmt.Sprintf("%016x", uint64(c))
 }
 
+// MarshalJSON implements the json.Marshaler interface for Checksum.
 func (c Checksum) MarshalJSON() ([]byte, error) {
 	return []byte(`"` + c.String() + `"`), nil
 }
 
-func (c *Checksum) UnmarshalJSON(data []byte) (err error) {
+// UnmarshalJSON implements the json.Unmarshaler interface for Checksum.
+func (c *Checksum) UnmarshalJSON(data []byte) error {
 	var s *string
 	if err := json.Unmarshal(data, &s); err != nil {
 		return fmt.Errorf("cannot unmarshal checksum from JSON value")

cmd/ltx/apply.go

@@ -1,3 +1,4 @@
+// Package main implements the ltx command-line tool for working with LTX files.
 package main
 
 import (
@@ -19,7 +20,7 @@ func NewApplyCommand() *ApplyCommand {
 }
 
 // Run executes the command.
-func (c *ApplyCommand) Run(ctx context.Context, args []string) (ret error) {
+func (c *ApplyCommand) Run(ctx context.Context, args []string) error {
 	fs := flag.NewFlagSet("ltx-apply", flag.ContinueOnError)
 	dbPath := fs.String("db", "", "database path")
 	fs.Usage = func() {
@@ -66,7 +67,11 @@ Arguments:
 	return dbFile.Close()
 }
 
-func (c *ApplyCommand) applyLTXFile(_ context.Context, dbFile *os.File, filename string) error {
+func (*ApplyCommand) applyLTXFile(ctx context.Context, dbFile *os.File, filename string) error {
+	// Check for context cancellation
+	if err := ctx.Err(); err != nil {
+		return err
+	}
 	ltxFile, err := os.Open(filename)
 	if err != nil {
 		return err

cmd/ltx/checksum.go

@@ -20,7 +20,7 @@ func NewChecksumCommand() *ChecksumCommand {
 }
 
 // Run executes the command.
-func (c *ChecksumCommand) Run(ctx context.Context, args []string) (ret error) {
+func (*ChecksumCommand) Run(_ context.Context, args []string) error {
 	fs := flag.NewFlagSet("ltx-checksum", flag.ContinueOnError)
 	fs.Usage = func() {
 		fmt.Println(`

cmd/ltx/dump.go

@@ -20,7 +20,7 @@ func NewDumpCommand() *DumpCommand {
 }
 
 // Run executes the command.
-func (c *DumpCommand) Run(ctx context.Context, args []string) (ret error) {
+func (*DumpCommand) Run(_ context.Context, args []string) error {
 	fs := flag.NewFlagSet("ltx-dump", flag.ContinueOnError)
 	fs.Usage = func() {
 		fmt.Println(`