Skip to content

Commit ce6f6a3

Browse files
jasnelljasnell
committed
src: initial localization support
This is a work in progress effort to introduce the *optional* ability to localize node's own outputs. Currently, Node (and V8) outputs english only error, debug and help output. This is a step towards allowing translated versions of node to be built. Currently, this is hidden behind the --with-intl=full-icu switch. This is because there are certain capabilities of ICU that are only enabled with the full data set. This is a work in progress that SHOULD NOT BE LANDED in master yet.
1 parent b52142f commit ce6f6a3

File tree

16 files changed

+631
-22
lines changed

16 files changed

+631
-22
lines changed

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ clean:
7070
@if [ -d out ]; then find out/ -name '*.o' -o -name '*.a' -o -name '*.d' | xargs rm -rf; fi
7171
-rm -rf node_modules
7272
@if [ -d deps/icu ]; then echo deleting deps/icu; rm -rf deps/icu; fi
73+
-rm -rf out/Release/obj/gen/noderestmp
7374
-rm -f test.tap
7475

7576
distclean:

‎deps/l10n/README.md‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
Localization bundle for Node.
2+
3+
This is pretty straightforward... if ICU is present and --with-intl-full-icu,
4+
then ICU's resource bundle mechanism is used, the resources are compiled
5+
statically into the library, which can then be used within Node. If ICU is not
6+
present, a simple printf fallback is used.
7+
8+
```
9+
./configure --with-intl=full-icu
10+
make
11+
```
12+
When the --with-intl=full-icu switch is on, the resources are compiled into a
13+
static library that is statically linked into Node. The next step will be to
14+
make it so that additional bundles can be specified at runtime.
15+
16+
Resource bundles are located in the resources directory. Standard ICU bundle
17+
format but keep it simple, we currently only support top level resources.
18+
19+
Within the C/C++ code, use the macros:
20+
21+
```cc
22+
#include "node_l10n.h"
23+
24+
L10N_PRINT("TEST", "This is the fallback");
25+
L10N_PRINTF("TEST2", "Testing %s %d", "a", 1);
26+
```
27+
28+
In the JS code, use the internal/l10n.js
29+
```javascript
30+
const l10n = require('internal/l10n');
31+
console.log(l10n("TEST", "This is the fallback"));
32+
console.log(l10n("TEST2", "Fallback %s %d", "a", 1));
33+
```
34+
35+
Use the `--icu-data-dir` switch to specify a location containing alternative
36+
node.dat files containing alternative translations. Note that this is the
37+
same switch used to specify alternative ICU common data files.
38+
39+
One approach that ought to work here is publishing translation node.dat files
40+
to npm. Then, the developer does a `npm install node_dat_de` (for instance)
41+
to grab the appropriate translations. Then, then can start node like:
42+
43+
`node --icu-data-dir=node_modules/node_dat_de` and have things just work.

‎deps/l10n/icures.py‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
#!/usr/bin/python
2+
3+
import sys
4+
import shutil
5+
reload(sys)
6+
sys.setdefaultencoding("utf-8")
7+
8+
import optparse
9+
import os
10+
import glob
11+
12+
endian=sys.byteorder
13+
14+
parser = optparse.OptionParser(usage="usage: %prog -n {NAME} -d {DEST} -i {ICU}")
15+
16+
parser.add_option("-d", "--dest-dir",
17+
action="store",
18+
dest="dest",
19+
help="The destination directory")
20+
21+
parser.add_option("-n", "--name",
22+
action="store",
23+
dest="name",
24+
help="The application package name")
25+
26+
parser.add_option("-i", "--icu-path",
27+
action="store",
28+
dest="icu",
29+
help="The ICU tool path")
30+
31+
parser.add_option("-l", "--endian",
32+
action="store",
33+
dest="endian",
34+
help='endian: big, little or host. your default is "%s"' % endian, default=endian, metavar='endianess')
35+
36+
(options, args) = parser.parse_args()
37+
38+
optVars = vars(options);
39+
40+
for opt in ["dest", "name", "icu"]:
41+
if optVars[opt] is None:
42+
parser.error("Missing required option: %s" % opt)
43+
sys.exit(1)
44+
45+
if options.endian not in ("big", "little", "host"):
46+
parser.error("Unknown endianess: %s" % options.endian)
47+
sys.exit(1)
48+
49+
if options.endian == "host":
50+
options.endian = endian
51+
52+
if not os.path.isdir(options.dest):
53+
parser.error("Destination is not a directory")
54+
sys.exit(1)
55+
56+
if options.icu[-1] is '"':
57+
options.icu = options.icu[:-1]
58+
59+
if not os.path.isdir(options.icu):
60+
parser.error("ICU Path is not a directory")
61+
sys.exit(1)
62+
63+
if options.icu[-1] != os.path.sep:
64+
options.icu += os.path.sep
65+
66+
genrb = options.icu + 'genrb'
67+
icupkg = options.icu + 'icupkg'
68+
69+
if sys.platform.startswith('win32'):
70+
genrb += '.exe'
71+
icupkg += '.exe'
72+
73+
if not os.path.isfile(genrb):
74+
parser.error('ICU Tool "%s" does not exist' % genrb)
75+
sys.exit(1)
76+
77+
if not os.path.isfile(icupkg):
78+
parser.error('ICU Tool "%s" does not exist' % icupkg)
79+
sys.exit(1)
80+
81+
def runcmd(tool, cmd, doContinue=False):
82+
cmd = "%s %s" % (tool, cmd)
83+
rc = os.system(cmd)
84+
if rc is not 0 and not doContinue:
85+
print "FAILED: %s" % cmd
86+
sys.exit(1)
87+
return rc
88+
89+
resfiles = glob.glob("%s%s*.res" % (options.dest, os.path.sep))
90+
_listfile = os.path.join(options.dest, 'packagefile.lst')
91+
datfile = "%s%s%s.dat" % (options.dest, os.path.sep, options.name)
92+
93+
def clean():
94+
for f in resfiles:
95+
if os.path.isfile(f):
96+
os.remove(f)
97+
if os.path.isfile(_listfile):
98+
os.remove(_listfile)
99+
if (os.path.isfile(datfile)):
100+
os.remove(datfile)
101+
102+
## Step 0, Clean up from previous build
103+
clean()
104+
105+
## Step 1, compile the txt files in res files
106+
107+
if sys.platform.startswith('win32'):
108+
srcfiles = glob.glob('resources/*.txt')
109+
runcmd(genrb, "-e utf16 -d %s %s" % (options.dest, " ".join(srcfiles)))
110+
else:
111+
runcmd(genrb, "-e utf16 -d %s resources%s*.txt" % (options.dest, os.path.sep))
112+
113+
resfiles = [os.path.relpath(f) for f in glob.glob("%s%s*.res" % (options.dest, os.path.sep))]
114+
115+
# pkgdata requires relative paths... it's annoying but necessary
116+
# for us to change into the dest directory to work
117+
cwd = os.getcwd();
118+
os.chdir(options.dest)
119+
120+
## Step 2, generate the package list
121+
listfile = open(_listfile, 'w')
122+
listfile.write(" ".join([os.path.basename(f) for f in resfiles]))
123+
listfile.close()
124+
125+
## Step 3, generate the dat file using icupkg and the package list
126+
runcmd(icupkg, '-a packagefile.lst new %s.dat' % options.name);
127+
128+
## All done with this tool at this point...
129+
os.chdir(cwd); # go back to original working directory

‎deps/l10n/include/l10n.h‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
#ifndef L10N__H
2+
#define L10N__H
3+
4+
#include <unicode/uloc.h>
5+
#include <unicode/ustdio.h>
6+
#include <unicode/ustring.h>
7+
8+
#ifdef _WIN32
9+
# define L10N_EXTERN /* nothing */
10+
#elif __GNUC__ >= 4
11+
# define L10N_EXTERN __attribute__((visibility("default")))
12+
#else
13+
# define L10N_EXTERN /* nothing */
14+
#endif
15+
16+
/**
17+
* Initialize the resource bundle. This will register an atexit handler
18+
* to deal with the cleanup in normal termination
19+
**/
20+
L10N_EXTERN bool l10n_initialize(const char * locale, const char * icu_data_dir);
21+
22+
/**
23+
* Lookup the key, return the value. Doesn't get much easier. If the key
24+
* is not found in the bundle, fallback is returned instead. The caller
25+
* owns the string and must delete[] it when done lest horrible things.
26+
**/
27+
L10N_EXTERN const uint16_t * l10n_fetch(const char * key,
28+
const uint16_t * fallback);
29+
30+
#define L10N(key, fallback) l10n_fetch(key, fallback)
31+
#define L10N_LOCALE() uloc_getDefault()
32+
#define L10N_INIT(locale, icu_data_dir) \
33+
do {l10n_initialize(locale, icu_data_dir);} while(0)
34+
35+
#endif // L10N__H

‎deps/l10n/l10n.gyp‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
{
2+
'includes': [ '../../icu_config.gypi' ],
3+
'targets': [
4+
{
5+
'target_name': 'noderes',
6+
'type': '<(library)',
7+
'conditions': [
8+
[
9+
'v8_enable_i18n_support==1',
10+
{
11+
'include_dirs': [ 'include', 'src' ],
12+
'direct_dependent_settings': {
13+
'include_dirs': [ 'include' ]
14+
},
15+
'sources': [
16+
'include/l10n.h',
17+
'include/l10n_version.h',
18+
'src/l10n.cc'
19+
],
20+
'dependencies': [
21+
'<(icu_gyp_path):icui18n',
22+
'<(icu_gyp_path):icuuc',
23+
'icu_noderes_data'
24+
]
25+
}
26+
]
27+
]
28+
},
29+
{
30+
#### build the resource bundle using ICU's tools ####
31+
'target_name': 'icu_noderes_data',
32+
'type': '<(library)',
33+
'conditions': [
34+
[
35+
'v8_enable_i18n_support==1',
36+
{
37+
'toolsets': [ 'target' ],
38+
'dependencies': [
39+
'<(icu_gyp_path):icu_implementation#host',
40+
'<(icu_gyp_path):icu_uconfig',
41+
'<(icu_gyp_path):genrb#host',
42+
'<(icu_gyp_path):genccode#host',
43+
'<(icu_gyp_path):icupkg#host'
44+
],
45+
'actions': [
46+
{
47+
'action_name': 'icures',
48+
'inputs': [
49+
'resources/root.txt',
50+
'resources/en.txt',
51+
'resources/en_US.txt'
52+
],
53+
'outputs': [
54+
'<(SHARED_INTERMEDIATE_DIR)/noderestmp/node.dat'
55+
],
56+
'action': [
57+
'python',
58+
'icures.py',
59+
'-n', 'node',
60+
'-d', '<(SHARED_INTERMEDIATE_DIR)/noderestmp',
61+
'-i', '<(PRODUCT_DIR)'
62+
]
63+
},
64+
{
65+
'action_name': 'icugen',
66+
'inputs': [
67+
'<(SHARED_INTERMEDIATE_DIR)/noderestmp/node.dat'
68+
],
69+
'conditions': [
70+
[
71+
'OS != "win"',
72+
{
73+
'outputs': [
74+
'<(SHARED_INTERMEDIATE_DIR)/node_dat.c'
75+
],
76+
'action': [
77+
'<(PRODUCT_DIR)/genccode',
78+
'-e', 'node',
79+
'-d', '<(SHARED_INTERMEDIATE_DIR)',
80+
'-f', 'node_dat',
81+
'<@(_inputs)'
82+
]
83+
},
84+
{
85+
'outputs': [
86+
'<(SHARED_INTERMEDIATE_DIR)/node_dat.obj'
87+
],
88+
'action': [
89+
'<(PRODUCT_DIR)/genccode',
90+
'-o',
91+
'-d', '<(SHARED_INTERMEDIATE_DIR)',
92+
'-n', 'node',
93+
'-e', 'node',
94+
'<@(_inputs)'
95+
]
96+
}
97+
]
98+
]
99+
}
100+
],
101+
'conditions': [
102+
[ 'OS == "win"',
103+
{
104+
'sources': [
105+
'<(SHARED_INTERMEDIATE_DIR)/node_dat.obj'
106+
]
107+
},
108+
{
109+
'include_dirs': [
110+
'../icu/source/common'
111+
],
112+
'sources': [
113+
'<(SHARED_INTERMEDIATE_DIR)/node_dat.c'
114+
]
115+
}
116+
]
117+
]
118+
}
119+
]
120+
]
121+
}
122+
]
123+
}

‎deps/l10n/resources/en.txt‎

72 Bytes
Binary file not shown.

‎deps/l10n/resources/en_US.txt‎

18 Bytes
Binary file not shown.

‎deps/l10n/resources/root.txt‎

114 Bytes
Binary file not shown.

‎deps/l10n/src/l10n.cc‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#include <stdlib.h>
2+
#include <unicode/udata.h>
3+
#include <unicode/ures.h>
4+
#include "l10n.h"
5+
6+
// The ICU bundle name
7+
#define L10N_APPDATA "node"
8+
9+
extern "C" const char U_DATA_API node_dat[];
10+
11+
UResourceBundle *bundle;
12+
13+
void l10n_cleanup() {
14+
ures_close(bundle);
15+
}
16+
17+
bool l10n_initialize(const char * locale,
18+
const char * icu_data_dir) {
19+
UErrorCode status = U_ZERO_ERROR;
20+
if (!icu_data_dir) {
21+
udata_setAppData("node", &node_dat, &status);
22+
}
23+
if (U_FAILURE(status)) {
24+
return FALSE;
25+
} else {
26+
bundle = ures_open(L10N_APPDATA, locale, &status);
27+
if (U_SUCCESS(status)) {
28+
atexit(l10n_cleanup);
29+
return TRUE;
30+
} else {
31+
return FALSE;
32+
}
33+
}
34+
}
35+
36+
const uint16_t * l10n_fetch(const char * key,
37+
const uint16_t * fallback) {
38+
UErrorCode status = U_ZERO_ERROR;
39+
int32_t len = 0;
40+
const UChar* res = ures_getStringByKey(
41+
bundle,
42+
key,
43+
&len,
44+
&status);
45+
const uint16_t* ret = static_cast<const uint16_t*>(res);
46+
if (U_SUCCESS(status)) {
47+
return ret;
48+
}
49+
return fallback;
50+
}

0 commit comments

Comments
 (0)