Cannot get File object of a subfolder created through google-cloud-node upload

[jduval]

After uploading a file with a specific destination, containing the subfolder name, I am not able to get the File object of this subfolder with Bucket.GetFiles().

However, if I look at the console, the folder is successfully created.

test/ is a folder create through the console, testSubFolder/ is created through google-cloud-node upload.

I uploaded the same file in each folder, and after a GetFiles(), here what I get :

// ...
metadata: 
  { kind: 'storage#object',
     id: 'xxxx/folder1/test//1473177306575000',
     selfLink: 'https://www.googleapis.com/storage/v1/b/xxxx/o/folder1%2Ftest%2F',
     name: 'folder1/test/',
// ...
metadata: 
  { kind: 'storage#object',
    id: 'xxxx/folder1/test/320x50.jpg/1473178787694000',
    selfLink: 'https://www.googleapis.com/storage/v1/b/xxxx/o/folder1%2Ftest%2F320x50.jpg',
     name: 'folder1/test/320x50.jpg',
// ...
metadata: 
  { kind: 'storage#object',
     id: 'xxxx/folder1/testSubFolder/320x50.jpg/1473178527390000',
     selfLink: 'https://www.googleapis.com/storage/v1/b/xxxx/o/folder1%2FtestSubFolder%2F320x50.jpg',
     name: 'folder1/testSubFolder/320x50.jpg',

the File object of folder1/testSubFolder/ is missing.

If I upload (with google-cloud-node) an emptyFile with a name ended by /, it successfully create a folder, and I can get its File object.

Environment details

• OS: ubuntu 14.04 LTS
• Node.js version: v6.2.0
• npm version: 3.8.9
• google-cloud-node version: 0.36.0

[stephenplusplus]

I think I'm either not following, or not able to reproduce. Can you run/modify this code to highlight the error?

$ npm install --save async
$ export GCLOUD_BUCKET=my-bucket-name

'use strict'

var async = require('async')
var storage = require('gcloud').storage()

var bucket = storage.bucket(process.env.GCLOUD_BUCKET)
var PREFIX = 'gcloud-issue-1570'

function createRandomFile(_, callback) {
  var file = bucket.file(PREFIX + '/' + Math.random() + '.txt')
  file.createWriteStream({ resumable: false })
    .on('error', callback)
    .on('finish', callback)
    .end('')
}

async.times(5, createRandomFile, function(err) {
  if (err) throw err

  bucket.getFiles(function(err, files) {
    if (err) throw err

    console.log(files.map(function(file) { return file.metadata.name }))

    bucket.deleteFiles({ prefix: PREFIX }, console.log)
  })
})

Thanks!

[jduval]

'use strict'

var async = require('async')
var storage = require('gcloud').storage()

var bucket = storage.bucket(process.env.GCLOUD_BUCKET)
var PREFIX = 'gcloud-issue-1570';

function createRandomFile(_, prefix, callback) {
  var file = bucket.file(prefix + '/' + Math.random() + '.txt');
  file.createWriteStream({ resumable: false })
    .on('error', callback)
    .on('finish', callback)
    .end('');
}

function createSubFolderFirst(callback) {
  var file = bucket.file('gcloud-issue-1570-subfolder-first/');
  file.createWriteStream({resumable: false})
  .on('error', callback)
  .on('finish', callback)
  .end('');
}

async.series([
  (cbk) => createSubFolderFirst(cbk),
  (cbk) =>
    async.times(5, (n, next) => createRandomFile(n, PREFIX, next), cbk),
  (cbk) =>
    async.times(5, (n, next) => createRandomFile(n, 'gcloud-issue-1570-subfolder-first', next), cbk),
], (err) => {
  if (err) throw err;

  bucket.getFiles(function(err, files) {
    if (err) throw err;

    console.log(files.map(function(file) { return file.metadata.name; }));

    bucket.deleteFiles({ prefix: PREFIX }, console.log);
  });
});

output :

[ 'gcloud-issue-1570-subfolder-first/', <-- File object when I create individually the folder
  'gcloud-issue-1570-subfolder-first/0.08110201042544585.txt',
  'gcloud-issue-1570-subfolder-first/0.11478644865661392.txt',
  'gcloud-issue-1570-subfolder-first/0.12316179549152517.txt',
  'gcloud-issue-1570-subfolder-first/0.22896304679202495.txt',
  'gcloud-issue-1570-subfolder-first/0.48825517360416737.txt',
  'gcloud-issue-1570/0.10549166177140101.txt', <-- missing when created through prefix with files
  'gcloud-issue-1570/0.12125933119508736.txt',
  'gcloud-issue-1570/0.6518799741675132.txt',
  'gcloud-issue-1570/0.6680695796204918.txt',
  'gcloud-issue-1570/0.9829666724113941.txt' ]

[stephenplusplus]

Okay, I think I'm following. In GCS, there are only "objects", there aren't actually "folders" or "folder objects". Having the / at the end of gcloud-issue-1570-subfolder-first/ is apparently enough to trigger its own Object to be created.

Here's a guide on how GCS subdirectories work-- basically, don't worry about creating a folder manually. It will be created for you if its part of the new object's file path at the time of uploading.