Skip to content

Commit 79a265b

Browse files
SheetJSDevSheetJSDev
committed
version bump 0.4.0
- added missing bitshift (fixes #5) - brute-force unicode tests
1 parent 74dda6d commit 79a265b

21 files changed

+182
-24
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
node_modules
2+
test_files/*.py
3+
test_files/*.js
4+
test_files/baseline*
25
misc/coverage.html
36
misc/*/

‎.npmignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,12 @@ misc/
33
perf/
44
bits/
55
ctest/
6+
test_files/
67
test.js
78
.travis.yml
89
.jscs.json
10+
.jshintrc
11+
.flowconfig
12+
.npmignore
913
perf.txt
1014
Makefile

‎.travis.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ node_js:
88
before_install:
99
- "npm install -g npm@next"
1010
- "npm install -g mocha crc-32 benchmark ansi"
11+
- "npm install codepage"
1112
- "npm install blanket"
1213
- "npm install coveralls mocha-lcov-reporter"
1314
after_success:

‎LICENSE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Copyright (C) 2014 SheetJS
1+
Copyright (C) 2014-present SheetJS
22

33
Licensed under the Apache License, Version 2.0 (the "License");
44
you may not use this file except in compliance with the License.

‎Makefile

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,11 @@ bits/01_version.js: package.json
2020
echo "CRC32.version = '"`grep version package.json | awk '{gsub(/[^0-9a-z\.-]/,"",$$2); print $$2}'`"';" > $@
2121

2222
.PHONY: clean
23-
clean:
23+
clean: clean-baseline
2424
rm -f $(TARGET)
2525

2626
.PHONY: test mocha
27-
test mocha: test.js
27+
test mocha: test.js $(TARGET) baseline
2828
mocha -R spec -t 20000
2929

3030
.PHONY: ctest
@@ -70,3 +70,10 @@ perf:
7070
.PHONY: perf-all
7171
perf-all:
7272
bash misc/perf.sh
73+
74+
.PHONY: baseline clean-baseline
75+
baseline:
76+
./misc/make_baseline.sh
77+
78+
clean-baseline:
79+
rm -f test_files/*.*

‎README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ For example:
5151
To run the in-browser tests, run a local server and go to the `ctest` directory.
5252
To update the browser artifacts, run `make ctest`.
5353

54+
`make baseline` will generate baseline files based on the unicode mapping at
55+
<http://mathias.html5.org>
56+
5457
## License
5558

5659
Please consult the attached LICENSE file for details. All rights not explicitly

‎bits/00_header.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* crc32.js (C) 2014-2015 SheetJS -- http://sheetjs.com */
1+
/* crc32.js (C) 2014-present SheetJS -- http://sheetjs.com */
22
/* vim: set ts=2: */
33
var CRC32;
44
/*:: declare var DO_NOT_EXPORT_CRC: any; */
@@ -14,7 +14,7 @@ var CRC32;
1414
return module;
1515
});
1616
} else {
17-
factory(CRC32 = {});
17+
factory(CRC32 = {});
1818
}
1919
} else {
2020
factory(CRC32 = {});

‎bits/01_version.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
CRC32.version = '0.3.0';
1+
CRC32.version = '0.4.0';

‎bits/40_crc.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ function crc32_str(str/*:string*/)/*:CRC32Type*/ {
5151
c = (c&1023)+64; d = str.charCodeAt(i++) & 1023;
5252
crc = (crc >>> 8) ^ table[(crc ^ (240|((c>>8)&7))) & 0xFF];
5353
crc = (crc >>> 8) ^ table[(crc ^ (128|((c>>2)&63))) & 0xFF];
54-
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|(c&3))) & 0xFF];
54+
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|((c&3)<<4))) & 0xFF];
5555
crc = (crc >>> 8) ^ table[(crc ^ (128|(d&63))) & 0xFF];
5656
} else {
5757
crc = (crc >>> 8) ^ table[(crc ^ (224|((c>>12)&15))) & 0xFF];

‎crc32.flow.js

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* crc32.js (C) 2014-2015 SheetJS -- http://sheetjs.com */
1+
/* crc32.js (C) 2014-present SheetJS -- http://sheetjs.com */
22
/* vim: set ts=2: */
33
var CRC32;
44
/*:: declare var DO_NOT_EXPORT_CRC: any; */
@@ -14,13 +14,13 @@ var CRC32;
1414
return module;
1515
});
1616
} else {
17-
factory(CRC32 = {});
17+
factory(CRC32 = {});
1818
}
1919
} else {
2020
factory(CRC32 = {});
2121
}
2222
}(function(CRC32) {
23-
CRC32.version = '0.3.0';
23+
CRC32.version = '0.4.0';
2424
/*::
2525
type CRC32Type = number;
2626
type ABuf = Array<number> | Buffer;
@@ -100,7 +100,7 @@ function crc32_str(str/*:string*/)/*:CRC32Type*/ {
100100
c = (c&1023)+64; d = str.charCodeAt(i++) & 1023;
101101
crc = (crc >>> 8) ^ table[(crc ^ (240|((c>>8)&7))) & 0xFF];
102102
crc = (crc >>> 8) ^ table[(crc ^ (128|((c>>2)&63))) & 0xFF];
103-
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|(c&3))) & 0xFF];
103+
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|((c&3)<<4))) & 0xFF];
104104
crc = (crc >>> 8) ^ table[(crc ^ (128|(d&63))) & 0xFF];
105105
} else {
106106
crc = (crc >>> 8) ^ table[(crc ^ (224|((c>>12)&15))) & 0xFF];

‎crc32.js

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* crc32.js (C) 2014-2015 SheetJS -- http://sheetjs.com */
1+
/* crc32.js (C) 2014-present SheetJS -- http://sheetjs.com */
22
/* vim: set ts=2: */
33
var CRC32;
44
(function (factory) {
@@ -12,13 +12,13 @@ var CRC32;
1212
return module;
1313
});
1414
} else {
15-
factory(CRC32 = {});
15+
factory(CRC32 = {});
1616
}
1717
} else {
1818
factory(CRC32 = {});
1919
}
2020
}(function(CRC32) {
21-
CRC32.version = '0.3.0';
21+
CRC32.version = '0.4.0';
2222
/* see perf/crc32table.js */
2323
function signed_crc_table() {
2424
var c = 0, table = new Array(256);
@@ -93,7 +93,7 @@ function crc32_str(str) {
9393
c = (c&1023)+64; d = str.charCodeAt(i++) & 1023;
9494
crc = (crc >>> 8) ^ table[(crc ^ (240|((c>>8)&7))) & 0xFF];
9595
crc = (crc >>> 8) ^ table[(crc ^ (128|((c>>2)&63))) & 0xFF];
96-
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|(c&3))) & 0xFF];
96+
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|((c&3)<<4))) & 0xFF];
9797
crc = (crc >>> 8) ^ table[(crc ^ (128|(d&63))) & 0xFF];
9898
} else {
9999
crc = (crc >>> 8) ^ table[(crc ^ (224|((c>>12)&15))) & 0xFF];

‎ctest/crc32.js

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* crc32.js (C) 2014-2015 SheetJS -- http://sheetjs.com */
1+
/* crc32.js (C) 2014-present SheetJS -- http://sheetjs.com */
22
/* vim: set ts=2: */
33
var CRC32;
44
(function (factory) {
@@ -12,13 +12,13 @@ var CRC32;
1212
return module;
1313
});
1414
} else {
15-
factory(CRC32 = {});
15+
factory(CRC32 = {});
1616
}
1717
} else {
1818
factory(CRC32 = {});
1919
}
2020
}(function(CRC32) {
21-
CRC32.version = '0.3.0';
21+
CRC32.version = '0.4.0';
2222
/* see perf/crc32table.js */
2323
function signed_crc_table() {
2424
var c = 0, table = new Array(256);
@@ -93,7 +93,7 @@ function crc32_str(str) {
9393
c = (c&1023)+64; d = str.charCodeAt(i++) & 1023;
9494
crc = (crc >>> 8) ^ table[(crc ^ (240|((c>>8)&7))) & 0xFF];
9595
crc = (crc >>> 8) ^ table[(crc ^ (128|((c>>2)&63))) & 0xFF];
96-
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|(c&3))) & 0xFF];
96+
crc = (crc >>> 8) ^ table[(crc ^ (128|((d>>6)&15)|((c&3)<<4))) & 0xFF];
9797
crc = (crc >>> 8) ^ table[(crc ^ (128|(d&63))) & 0xFF];
9898
} else {
9999
crc = (crc >>> 8) ^ table[(crc ^ (224|((c>>12)&15))) & 0xFF];

‎ctest/test.js

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@ if(typeof require !== 'undefined') {
55
describe('source',function(){it('should load',function(){X=require('./');});});
66
bits = require('./misc/bits.js');
77
crc32table = require('./misc/table.js');
8+
fs = require("fs");
89
} else { X = CRC32; }
910

11+
function readlines(f) { return fs.readFileSync(f, "ascii").split("\n").filter(function(f) { return !!f; }); }
12+
1013
describe('crc32 table', function() {
1114
it('should match fixed table', function() {
1215
var badness = 0;
@@ -30,4 +33,20 @@ describe('crc32 bits', function() {
3033
});
3134
});
3235
});
33-
36+
if(typeof require !== 'undefined') describe("unicode", function() {
37+
if(!fs.existsSync("./test_files/uccat.txt")) return;;
38+
var uccat = readlines("./test_files/uccat.txt");
39+
uccat.forEach(function(cat) {
40+
it("Category " + cat, function() {
41+
if(!fs.existsSync("./test_files/baseline." + cat + ".txt")) return;
42+
var corpus = readlines("./test_files/baseline." + cat + ".txt");
43+
var uctable = require("./test_files/uctable." + cat + ".js");
44+
uctable.forEach(function(c, i) {
45+
/* since the baselines are passed via utf8, discard invalid codes */
46+
if(c.charCodeAt(0) >= 0xD800 && c.charCodeAt(0) < 0xE000) return;
47+
var cc = corpus[i], dd = X.str(c);
48+
assert.equal(dd, cc, ":" + i + ":" + c + ":" + cc + ":" + dd);
49+
});
50+
});
51+
});
52+
});

‎misc/make_baseline.sh

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# make_baseline.sh -- generate baselines for tests
3+
# Copyright (C) 2016-present SheetJS
4+
OUTD=../test_files
5+
CATURL=https://mathias.html5.org/data/unicode/8.0.0/categories/
6+
CATF=$OUTD/uccat.txt
7+
8+
ECHORED() { echo -ne '\x1B[0;31m'; echo -n $1; echo -ne '\x1B[0m'; echo; }
9+
10+
if [ -d misc ]; then cd misc; fi
11+
mkdir -p $OUTD
12+
if [ ! -e $CATF ]; then curl "$CATURL" | grep "code-points" | sed 's/.*="//g;s/-.*//g' > $CATF; fi
13+
14+
while read line; do
15+
JSF=uctable.${line}.js
16+
PYF=uctable_${line}.py
17+
BLF=baseline.${line}.txt
18+
JSURL="https://mathias.html5.org/data/unicode/format?version=8.0.0&category=${line}&type=symbols&prepend=var+unicode%20%3D%20&append=%3Bif(typeof%20module%20!%3D%3D%20'undefined')%20module.exports%20%3D%20unicode%3B"
19+
if [[ ! -e $OUTD/$JSF || ! -e $OUTD/$PYF || ! -e $OUTD/$BLF ]]; then
20+
ECHORED "Processing ${line}"
21+
if [ ! -e $JSF ]; then
22+
rm -f $PYF $BLF ${PYF}c
23+
echo "Downloading JS"
24+
</dev/null curl -o $JSF "$JSURL"
25+
fi
26+
if [ ! -e $PYF ]; then
27+
echo "Building Python script"
28+
rm -f $BLF ${PYF}c
29+
</dev/null node make_unicode_crc.njs ${line} | sed 's/\[ \[/uctable = \[ \[/' > $PYF
30+
fi
31+
if [ ! -e $BLF ]; then
32+
echo "Building Baseline text"
33+
python make_unicode_crc.py ${line} > baseline.${line}.txt
34+
fi
35+
for i in $JSF $PYF $BLF; do if [ -e $i ]; then mv $i $OUTD/; fi; done
36+
rm -f uctable_${line}.pyc
37+
fi
38+
done < $CATF
39+

‎misc/make_unicode_crc.njs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env node
2+
argv = process.argv.slice(2);
3+
var enc = require('codepage').utils.encode;
4+
function arr(x) { return [].slice.call(enc(65001, x)); }
5+
console.log(require('./uctable.' + argv[0]).map(arr));

‎misc/make_unicode_crc.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env python
2+
# make_unicode_crc.py -- generate baselines for tests
3+
# Copyright (C) 2016-present SheetJS
4+
5+
from zlib import crc32
6+
from array import array
7+
from sys import argv, stderr, exit
8+
from importlib import import_module
9+
10+
args = argv[1:]
11+
12+
if len(args) < 1:
13+
print >>stderr, "usage: " + argv[0] + " <category>"
14+
exit(1)
15+
16+
uctable = import_module("uctable_" + args[0]).uctable
17+
18+
for z in uctable:
19+
print crc32(array('B', z));

‎misc/spin.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/bash
22
# spin.sh -- show a spinner (for coverage test)
3-
# Copyright (C) 2014 SheetJS
3+
# Copyright (C) 2014-present SheetJS
44

55
wpid=$1
66
delay=1

‎package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{
22
"name": "crc-32",
3-
"version": "0.3.0",
3+
"version": "0.4.0",
44
"author": "sheetjs",
55
"description": "Pure-JS CRC-32",
66
"keywords": [ "crc32", "checksum", "crc" ],
77
"main": "./crc32",
88
"devDependencies": {
9+
"codepage":"",
910
"mocha":"",
1011
"uglify-js":""
1112
},

‎perf/bm.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/* bm.js (C) 2014 SheetJS -- http://sheetjs.com */
1+
/* bm.js (C) 2014-present SheetJS -- http://sheetjs.com */
22
var Benchmark = require('benchmark');
33
var c = require('ansi')(process.stdout);
44

‎test.js

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@ if(typeof require !== 'undefined') {
55
describe('source',function(){it('should load',function(){X=require('./');});});
66
bits = require('./misc/bits.js');
77
crc32table = require('./misc/table.js');
8+
fs = require("fs");
89
} else { X = CRC32; }
910

11+
function readlines(f) { return fs.readFileSync(f, "ascii").split("\n").filter(function(f) { return !!f; }); }
12+
1013
describe('crc32 table', function() {
1114
it('should match fixed table', function() {
1215
var badness = 0;
@@ -30,4 +33,20 @@ describe('crc32 bits', function() {
3033
});
3134
});
3235
});
33-
36+
if(typeof require !== 'undefined') describe("unicode", function() {
37+
if(!fs.existsSync("./test_files/uccat.txt")) return;;
38+
var uccat = readlines("./test_files/uccat.txt");
39+
uccat.forEach(function(cat) {
40+
it("Category " + cat, function() {
41+
if(!fs.existsSync("./test_files/baseline." + cat + ".txt")) return;
42+
var corpus = readlines("./test_files/baseline." + cat + ".txt");
43+
var uctable = require("./test_files/uctable." + cat + ".js");
44+
uctable.forEach(function(c, i) {
45+
/* since the baselines are passed via utf8, discard invalid codes */
46+
if(c.charCodeAt(0) >= 0xD800 && c.charCodeAt(0) < 0xE000) return;
47+
var cc = corpus[i], dd = X.str(c);
48+
assert.equal(dd, cc, ":" + i + ":" + c + ":" + cc + ":" + dd);
49+
});
50+
});
51+
});
52+
});

‎test_files/uccat.txt

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
C
2+
Cc
3+
Cf
4+
Cn
5+
Co
6+
Cs
7+
L
8+
LC
9+
Ll
10+
Lm
11+
Lo
12+
Lt
13+
Lu
14+
M
15+
Mc
16+
Me
17+
Mn
18+
N
19+
Nd
20+
Nl
21+
No
22+
P
23+
Pc
24+
Pd
25+
Pe
26+
Pf
27+
Pi
28+
Po
29+
Ps
30+
S
31+
Sc
32+
Sk
33+
Sm
34+
So
35+
Z
36+
Zl
37+
Zp
38+
Zs

0 commit comments

Comments
 (0)