Skip to content

Commit 105fe14

Browse files
jf-techjf-tech
authored
Complete the edi fileformat (#105)
1 parent a705bb0 commit 105fe14

File tree

7 files changed

+508
-22
lines changed

7 files changed

+508
-22
lines changed

‎extensions/omniv21/fileformat/edi/.snapshots/TestValidateSchema-success

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"file_declaration": {
3+
"segment_delimiter": "\n",
4+
"element_delimiter": "*",
5+
"segment_declarations": [
6+
{
7+
"name": "ISA",
8+
"is_target": true,
9+
"elements": [
10+
{
11+
"name": "e1",
12+
"index": 1
13+
},
14+
{
15+
"name": "e2",
16+
"index": 2
17+
}
18+
]
19+
}
20+
]
21+
},
22+
"XPath": "."
23+
}

‎extensions/omniv21/fileformat/edi/format.go

+84
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package edi
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"io"
7+
"strings"
8+
9+
"github.com/jf-tech/go-corelib/caches"
10+
"github.com/jf-tech/go-corelib/strs"
11+
12+
"github.com/jf-tech/omniparser/errs"
13+
"github.com/jf-tech/omniparser/extensions/omniv21/fileformat"
14+
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
15+
v21validation "github.com/jf-tech/omniparser/extensions/omniv21/validation"
16+
"github.com/jf-tech/omniparser/validation"
17+
)
18+
19+
const (
20+
fileFormatEDI = "edi"
21+
)
22+
23+
type ediFileFormat struct {
24+
schemaName string
25+
}
26+
27+
// NewEDIFileFormat creates a FileFormat for EDI.
28+
func NewEDIFileFormat(schemaName string) fileformat.FileFormat {
29+
return &ediFileFormat{schemaName: schemaName}
30+
}
31+
32+
type ediFormatRuntime struct {
33+
Decl *fileDecl `json:"file_declaration"`
34+
XPath string
35+
}
36+
37+
func (f *ediFileFormat) ValidateSchema(
38+
format string, schemaContent []byte, finalOutputDecl *transform.Decl) (interface{}, error) {
39+
if format != fileFormatEDI {
40+
return nil, errs.ErrSchemaNotSupported
41+
}
42+
err := validation.SchemaValidate(f.schemaName, schemaContent, v21validation.JSONSchemaEDIFileDeclaration)
43+
if err != nil {
44+
// err is already context formatted.
45+
return nil, err
46+
}
47+
var runtime ediFormatRuntime
48+
_ = json.Unmarshal(schemaContent, &runtime) // JSON schema validation earlier guarantees Unmarshal success.
49+
err = f.validateFileDecl(runtime.Decl)
50+
if err != nil {
51+
// err is already context formatted.
52+
return nil, err
53+
}
54+
if finalOutputDecl == nil {
55+
return nil, f.FmtErr("'FINAL_OUTPUT' is missing")
56+
}
57+
runtime.XPath = strings.TrimSpace(strs.StrPtrOrElse(finalOutputDecl.XPath, ""))
58+
if runtime.XPath != "" {
59+
_, err := caches.GetXPathExpr(runtime.XPath)
60+
if err != nil {
61+
return nil, f.FmtErr("'FINAL_OUTPUT.xpath' (value: '%s') is invalid, err: %s",
62+
runtime.XPath, err.Error())
63+
}
64+
}
65+
return &runtime, nil
66+
}
67+
68+
func (f *ediFileFormat) validateFileDecl(decl *fileDecl) error {
69+
err := (&ediValidateCtx{}).validateFileDecl(decl)
70+
if err != nil {
71+
return f.FmtErr(err.Error())
72+
}
73+
return err
74+
}
75+
76+
func (f *ediFileFormat) CreateFormatReader(
77+
name string, r io.Reader, runtime interface{}) (fileformat.FormatReader, error) {
78+
edi := runtime.(*ediFormatRuntime)
79+
return NewReader(name, r, edi.Decl, edi.XPath)
80+
}
81+
82+
func (f *ediFileFormat) FmtErr(format string, args ...interface{}) error {
83+
return fmt.Errorf("schema '%s': %s", f.schemaName, fmt.Sprintf(format, args...))
84+
}

‎extensions/omniv21/fileformat/edi/format_test.go

+189
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package edi
2+
3+
import (
4+
"io"
5+
"strings"
6+
"testing"
7+
8+
"github.com/bradleyjkemp/cupaloy"
9+
"github.com/jf-tech/go-corelib/jsons"
10+
"github.com/jf-tech/go-corelib/strs"
11+
"github.com/stretchr/testify/assert"
12+
13+
"github.com/jf-tech/omniparser/extensions/omniv21/transform"
14+
"github.com/jf-tech/omniparser/idr"
15+
)
16+
17+
func TestValidateSchema(t *testing.T) {
18+
for _, test := range []struct {
19+
name string
20+
format string
21+
fileDecl string
22+
finalOutput *transform.Decl
23+
err string
24+
}{
25+
{
26+
name: "format not supported",
27+
format: "mp3",
28+
fileDecl: "",
29+
finalOutput: nil,
30+
err: "schema not supported",
31+
},
32+
{
33+
name: "json schema validation fail",
34+
format: fileFormatEDI,
35+
fileDecl: `{
36+
"file_declaration": {
37+
"segment_delimiter": "\n",
38+
"element_delimiter": "*",
39+
"segment_declarations": [
40+
{
41+
"name": "ISA",
42+
"is_target": true,
43+
"max": -2,
44+
"elements": [
45+
{ "name": "e1", "index": 1 }
46+
]
47+
}
48+
]
49+
}
50+
}`,
51+
finalOutput: nil,
52+
err: `schema 'test' validation failed: file_declaration.segment_declarations.0.max: Must be greater than or equal to -1`,
53+
},
54+
{
55+
name: "in code schema validation fail",
56+
format: fileFormatEDI,
57+
fileDecl: `{
58+
"file_declaration": {
59+
"segment_delimiter": "\n",
60+
"element_delimiter": "*",
61+
"segment_declarations": [
62+
{
63+
"name": "ISA",
64+
"is_target": true,
65+
"max": 0,
66+
"elements": [
67+
{ "name": "e1", "index": 1 }
68+
]
69+
}
70+
]
71+
}
72+
}`,
73+
finalOutput: nil,
74+
err: `schema 'test': segment 'ISA' has 'min' value 1 > 'max' value 0`,
75+
},
76+
{
77+
name: "FINAL_OUTPUT is nil",
78+
format: fileFormatEDI,
79+
fileDecl: `{
80+
"file_declaration": {
81+
"segment_delimiter": "\n",
82+
"element_delimiter": "*",
83+
"segment_declarations": [
84+
{
85+
"name": "ISA",
86+
"is_target": true,
87+
"elements": [
88+
{ "name": "e1", "index": 1 }
89+
]
90+
}
91+
]
92+
}
93+
}`,
94+
finalOutput: nil,
95+
err: `schema 'test': 'FINAL_OUTPUT' is missing`,
96+
},
97+
{
98+
name: "FINAL_OUTPUT xpath is invalid",
99+
format: fileFormatEDI,
100+
fileDecl: `{
101+
"file_declaration": {
102+
"segment_delimiter": "\n",
103+
"element_delimiter": "*",
104+
"segment_declarations": [
105+
{
106+
"name": "ISA",
107+
"is_target": true,
108+
"elements": [
109+
{ "name": "e1", "index": 1 }
110+
]
111+
}
112+
]
113+
}
114+
}`,
115+
finalOutput: &transform.Decl{XPath: strs.StrPtr("[")},
116+
err: `schema 'test': 'FINAL_OUTPUT.xpath' (value: '[') is invalid, err: expression must evaluate to a node-set`,
117+
},
118+
{
119+
name: "success",
120+
format: fileFormatEDI,
121+
fileDecl: `{
122+
"file_declaration": {
123+
"segment_delimiter": "\n",
124+
"element_delimiter": "*",
125+
"segment_declarations": [
126+
{
127+
"name": "ISA",
128+
"is_target": true,
129+
"elements": [
130+
{ "name": "e2", "index": 2 },
131+
{ "name": "e1", "index": 1 }
132+
]
133+
}
134+
]
135+
}
136+
}`,
137+
finalOutput: &transform.Decl{XPath: strs.StrPtr(".")},
138+
err: ``,
139+
},
140+
} {
141+
t.Run(test.name, func(t *testing.T) {
142+
rt, err := NewEDIFileFormat("test").ValidateSchema(test.format, []byte(test.fileDecl), test.finalOutput)
143+
if test.err != "" {
144+
assert.Error(t, err)
145+
assert.Equal(t, test.err, err.Error())
146+
assert.Nil(t, rt)
147+
} else {
148+
assert.NoError(t, err)
149+
cupaloy.SnapshotT(t, jsons.BPM(rt))
150+
}
151+
})
152+
}
153+
}
154+
155+
func TestCreateFormatReader(t *testing.T) {
156+
format := NewEDIFileFormat("test")
157+
fileDecl := `{
158+
"file_declaration": {
159+
"segment_delimiter": "\n",
160+
"element_delimiter": "*",
161+
"segment_declarations": [
162+
{
163+
"name": "ISA",
164+
"is_target": true,
165+
"elements": [
166+
{ "name": "e3", "index": 3 },
167+
{ "name": "e1", "index": 1 }
168+
]
169+
}
170+
]
171+
}
172+
}`
173+
rt, err := format.ValidateSchema(fileFormatEDI, []byte(fileDecl), &transform.Decl{XPath: strs.StrPtr(".")})
174+
assert.NoError(t, err)
175+
reader, err := format.CreateFormatReader("test", strings.NewReader("ISA*e1*e2*e3\nISA*e4*e5*e6\n"), rt)
176+
assert.NoError(t, err)
177+
n, err := reader.Read()
178+
assert.NoError(t, err)
179+
assert.Equal(t, `{"e1":"e1","e3":"e3"}`, idr.JSONify2(n))
180+
reader.Release(n)
181+
n, err = reader.Read()
182+
assert.NoError(t, err)
183+
assert.Equal(t, `{"e1":"e4","e3":"e6"}`, idr.JSONify2(n))
184+
reader.Release(n)
185+
n, err = reader.Read()
186+
assert.Error(t, err)
187+
assert.Equal(t, io.EOF, err)
188+
assert.Nil(t, n)
189+
}

‎extensions/omniv21/fileformat/edi/reader.go

+16-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package edi
33
import (
44
"bufio"
55
"bytes"
6+
"errors"
67
"fmt"
78
"io"
89
"unicode/utf8"
@@ -250,7 +251,6 @@ func (r *ediReader) rawSegToNode(segDecl *segDecl) (*idr.Node, error) {
250251
}
251252
n := idr.CreateNode(idr.ElementNode, segDecl.Name)
252253
// Note: we assume segDecl.Elems are sorted by elemIndex/compIndex.
253-
// TODO: do the sorting validation.
254254
rawElemIndex := 0
255255
rawElems := r.unprocessedRawSeg.elems
256256
for _, elemDecl := range segDecl.Elems {
@@ -405,6 +405,21 @@ func (r *ediReader) Read() (*idr.Node, error) {
405405
}
406406
}
407407

408+
func (r *ediReader) Release(n *idr.Node) {
409+
if r.target == n {
410+
r.target = nil
411+
}
412+
idr.RemoveAndReleaseTree(n)
413+
}
414+
415+
func (r *ediReader) IsContinuableError(err error) bool {
416+
return !IsErrInvalidEDI(err) && err != io.EOF
417+
}
418+
419+
func (r *ediReader) FmtErr(format string, args ...interface{}) error {
420+
return errors.New(r.fmtErrStr(format, args...))
421+
}
422+
408423
func (r *ediReader) fmtErrStr(format string, args ...interface{}) string {
409424
return fmt.Sprintf("input '%s' between character [%d,%d]: %s",
410425
r.inputName, r.runeBegin, r.runeEnd, fmt.Sprintf(format, args...))

0 commit comments

Comments
 (0)