diff --git a/internal/count/count.go b/internal/count/count.go
index be72f2d..b1a6e2d 100644
--- a/internal/count/count.go
+++ b/internal/count/count.go
@@ -152,10 +152,13 @@ func worker(counts chan<- Result, files <-chan string, wg *sync.WaitGroup) {
 	for file := range files {
 		f, err := os.Open(file)
 		if err != nil {
-			// If we can't open the file, just put an empty count on the results
-			// channel and move on
-			// TODO: Could probably improve this
-			counts <- Result{Name: file}
+			// If we can't open the (possibly dir) path
+			// then just continue
+			continue
+		}
+		info, _ := f.Stat() //nolint: errcheck // The file is already open here so we can ignore the error
+		// Skip directories
+		if info.IsDir() {
 			continue
 		}
 		result, err := One(f, file)
diff --git a/internal/count/count_test.go b/internal/count/count_test.go
index 6f30e21..03fc30b 100644
--- a/internal/count/count_test.go
+++ b/internal/count/count_test.go
@@ -113,6 +113,7 @@ func TestCountAll(t *testing.T) {
 		filepath.Join("testdata", "moby_dick.txt"),
 		filepath.Join("testdata", "another.txt"),
 		filepath.Join("testdata", "onemore.txt"),
+		filepath.Join("testdata", "dir"),
 	}
 
 	results, err := count.All(files)
diff --git a/internal/count/testdata/dir/.gitkeep b/internal/count/testdata/dir/.gitkeep
new file mode 100644
index 0000000..e69de29