Skip to content

Commit 2cc8f05

Browse files
eduard93eduard93
authored
Merge pull request #4 from evshvarov/master
UDL Classess
2 parents 32b1a2e + be5cf62 commit 2cc8f05

File tree

3 files changed

+582
-0
lines changed

3 files changed

+582
-0
lines changed

‎.DS_Store

6 KB
Binary file not shown.

‎src/cls/CacheUpdater/Task.cls

+245
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
Class CacheUpdater.Task Extends (%SYS.Task.Definition, CacheUpdater.UDL)
2+
{
3+
4+
Parameter TaskName = "GitHub Update";
5+
6+
/// Repository URL, like https://github.com/intersystems-ru/Cache-MDX2JSON
7+
/// Increased to 500 to support long urls
8+
Property GitHubURL As %String(MAXLEN = 500);
9+
10+
/// GitHub user, who has access to repository. Optional for public repositories.<br>
11+
/// Note, that with Username/Password, you can make up to 5,000 requests per hour.
12+
/// For unauthenticated requests, the rate limit allows to make up to 60 requests per hour.
13+
/// Unauthenticated requests are associated with an IP address.<br>
14+
/// Required, if you want to create webhooks
15+
Property Username As %String;
16+
17+
/// GitHub password, corresponding to Username. Optional for public repositories.
18+
Property Password As %String;
19+
20+
/// Namespace, where to download and compile repository
21+
Property Namespace As %String [ InitialExpression = {$Namespace} ];
22+
23+
/// Repository branch, usually master. Leave empty, if you want to receive default branch.
24+
Property Branch As %String [ InitialExpression = "master" ];
25+
26+
Method OnTask() As %Status
27+
{
28+
Return:'##class(%SYS.Namespace).Exists(..Namespace) $$$ERROR($$$NamespaceUnavailable,..Namespace)
29+
30+
Set Owner = $p(..GitHubURL,"/",4)
31+
Set Repository = $p(..GitHubURL,"/",5)
32+
33+
Return ..Update(Owner, Repository, ..Branch, ..Username, ..Password, ..Namespace)
34+
}
35+
36+
/// Downloads and compiles GitHub repository.<br>
37+
/// <b>Owner</b> - The name of the repository owner.<br>
38+
/// <b>Repository</b> - The name of the repository.<br>
39+
/// <b>Branch</b> - The name of the commit/branch/tag. If skipped the repository’s default branch (usually master) would be used.<br>
40+
/// <b>Username</b> - GitHub user, who has access to repository. Optional for public repositories.<br>
41+
/// <b>Password</b> - GitHub password, corresponding to Username. Optional for public repositories.<br>
42+
/// Note, that with Username, you can make up to 5,000 requests per hour.
43+
/// For unauthenticated requests, the rate limit allows to make up to 60 requests per hour.
44+
/// Unauthenticated requests are associated with an IP address.<br>
45+
/// <b>Namespace</b> - Namespace, where to download and compile repository.<br>
46+
///
47+
/// For example in the repository: https://github.com/intersystems-ru/Cache-MDX2JSON<br>
48+
/// Owner - intersystems-ru, Repository - Cache-MDX2JSON.<br>
49+
ClassMethod Update(Owner As %String, Repository As %String, Branch As %String = "", Username As %String = "", Password As %String = "", Namespace = {$Namespace}) As %Status
50+
{
51+
#dim req As %Net.HttpRequest
52+
Set req = ..CreateRequest(Username, Password)
53+
Set req.Location = "repos/" _ Owner _ "/" _ Repository _ "/contents" // as described in https://developer.github.com/v3/repos/
54+
55+
Set links = ##class(%ListOfDataTypes).%New()
56+
Set st = ..ProcessDirectory("",.req,Branch,.links)
57+
Return:$$$ISERR(st) st
58+
59+
Set namespace = $Namespace
60+
Zn Namespace
61+
Set st = ..DownloadFiles(links,req,.list)
62+
zw list
63+
Set st2 = $system.OBJ.CompileList(.list,"cuk /checkuptodate=expandedonly")
64+
Zn namespace
65+
66+
Return $$$ADDSC(st, st2)
67+
}
68+
69+
/// Process one directory of GitHub repository. Recursive.<br>
70+
/// <b>Path</b> -Internal repository path. Root is empty string<br>
71+
/// <b>Request</b> - Authenticated/Set %Net.HttpRequest object.<br>
72+
/// <b>Links</b> - List of links to raw files (which satisfy <b>IsCacheFile</b> conditions) from repository.<br>
73+
ClassMethod ProcessDirectory(Path As %String = "", Request As %Net.HttpRequest, Branch As %String = "", ByRef Links As %ListOfDataTypes) As %Status
74+
{
75+
Set location = Request.Location
76+
Set Request.Location = Request.Location _ Path
77+
Do:(Branch'="") Request.SetParam("ref",Branch)
78+
79+
Set st = Request.Get()
80+
81+
Return:$$$ISERR(st) st
82+
Return:(Request.HttpResponse.StatusCode = 404) $$$ERROR($$$GeneralError,"Repository doesn't exist OR you don't have access")
83+
Return:((Request.HttpResponse.StatusCode = 403) && (Request.HttpResponse.GetHeader("X-RATELIMIT-REMAINING")=0)) $$$ERROR($$$GeneralError,"API rate limit exceeded. Try logging in.")
84+
Return:(Request.HttpResponse.StatusCode '= 200) $$$ERROR($$$GeneralError,"Received " _ Request.HttpResponse.StatusCode _ " expected 200")
85+
86+
#dim objects As List of %ZEN.proxyObject
87+
#dim obj As %ZEN.proxyObject
88+
Set st = ##class(%ZEN.Auxiliary.jsonProvider).%ConvertJSONToObject(Request.HttpResponse.Data,,.objects,1)
89+
Return:$$$ISERR(st) st
90+
91+
For i = 1:1:objects.Count() {
92+
Set obj = objects.GetAt(i)
93+
If (obj.type = "dir") {
94+
Set st = ..ProcessDirectory("/"_obj.name,Request,Branch,.Links)
95+
Return:$$$ISERR(st) st
96+
} ElseIf (obj.type = "file") {
97+
//Do:..IsCacheFile(obj) Links.Insert(obj."download_url")
98+
Do Links.Insert($LB(obj."download_url",..IsCacheFile(obj)))
99+
} Else {
100+
// obj.type = "symlink" or obj.type = "submodule"
101+
}
102+
}
103+
Set Request.Location = location // to keep track of where in the repository tree we are
104+
Return $$$OK
105+
}
106+
107+
/// Check that incoming file is the one you need.
108+
ClassMethod IsCacheFile(File As %ZEN.proxyObject) As %Boolean
109+
{
110+
Set extensions = ",xml,cls,csp,csr,mac,int,bas,inc,gbl,prj,obj,pkg,gof,dfi,pivot,dashboard,html,css,js,ts,scss,"
111+
Return:($L(File.name,".")=1) 0 //no extension
112+
Set File.Extension = $P(File.name,".",$L(File.name,"."))
113+
Return $F(extensions,","_$ZCVT(File.Extension,"l")_",")
114+
}
115+
116+
/// Download list of files on https://raw.githubusercontent.com/ server.<br>
117+
/// <b>Links</b> - List of links to raw files.<br>
118+
/// <b>Request</b> - Authenticated/Set %Net.HttpRequest object.<br>
119+
/// <b>loadedlist</b> - Returns an array of the items loaded.
120+
ClassMethod DownloadFiles(Links As %ListOfDataTypes, Request As %Net.HttpRequest, Output Items) As %Status
121+
{
122+
Kill Items
123+
Set Request.Server = "raw.githubusercontent.com"
124+
Set st = $$$OK
125+
Try
126+
{
127+
For i = 1:1:Links.Count()
128+
{
129+
Set link = $ListGet(Links.GetAt(i),1)
130+
Set bIsCacheFile = $ListGet(Links.GetAt(i),2)
131+
Set ^gitfiles(i,"link")=link
132+
Set ^gitfiles(i,"bIsCacheFile")=bIsCacheFile
133+
134+
Set streq = Request.Get($e(link,35,*)) // Remove "https://raw.githubusercontent.com/" from URL.
135+
If $$$ISERR(streq)
136+
{
137+
Set st=$$$ADDSC(st, streq)
138+
Set ^gitfiles(i,"streq")=streq
139+
Continue
140+
}
141+
142+
Set ^gitfiles(i,"stream")="starting..."
143+
Set binarystream = Request.HttpResponse.Data
144+
145+
Do binarystream.Rewind() // just in case
146+
147+
Set characterStream=##class(%GlobalCharacterStream).%New() //translating binary stream into character stream
148+
Set stTranslate=$$$OK
149+
Try
150+
{
151+
While 'binarystream.AtEnd
152+
{
153+
//Use eol to prevent breaking lines larger than 32Kb
154+
Set line=binarystream.ReadLine(, .stTranslate, .eol)
155+
Quit:$System.Status.IsError(stTranslate)
156+
157+
If eol
158+
{
159+
Set stTranslate=characterStream.WriteLine(line)
160+
}
161+
Else
162+
{
163+
Set stTranslate=characterStream.Write(line)
164+
}
165+
Quit:$System.Status.IsError(stTranslate)
166+
}
167+
Quit:$System.Status.IsError(stTranslate)
168+
169+
Do characterStream.Rewind()
170+
}
171+
Catch (oTranslateStreamException)
172+
{
173+
Set stTranslate=oTranslateStreamException.AsStatus()
174+
}
175+
176+
If $System.Status.IsError(stTranslate)
177+
{
178+
//Could not convert binary stream to character stream
179+
//It is probably a binary file anyway
180+
Set characterStream=""
181+
Set st=$$$ADDSC(st, stTranslate)
182+
Set ^gitfiles(i,"stTranslate")=stTranslate
183+
}
184+
Set ^gitfiles(i,"stream")="Done"
185+
186+
Do binarystream.Rewind()
187+
188+
Set stload = $$$OK
189+
190+
set items = ""
191+
If ('$IsObject(characterStream)) || (..IsUDLFile(characterStream))
192+
{
193+
Set ^gitfiles(i,"IsUDLFile")="1"
194+
Set stload = ..LoadUDLFile(characterStream, binarystream, link, .items)
195+
}
196+
ElseIf bIsCacheFile
197+
{
198+
Set ^gitfiles(i,"IsUDLFile")="0"
199+
Set stload = $system.OBJ.LoadStream(characterStream,"",.error,.items,,,,"UTF8")
200+
}
201+
Set ^gitfiles(i,"stload")=stload
202+
If $$$ISERR(stload)
203+
{
204+
Set st=$$$ADDSC(st, stload)
205+
Continue
206+
}
207+
Merge Items = items // Does not overwrite existing array keys: Items(itemname)=""
208+
}
209+
210+
Set Request.Server="api.github.com"
211+
}
212+
Catch (oException)
213+
{
214+
Set st = oException.AsStatus()
215+
If $D(i) Set ^gitfiles(i,"st final")=st
216+
}
217+
218+
Quit st
219+
}
220+
221+
ClassMethod CreateRequest(Username As %String, Password As %String) As %Net.HttpRequest
222+
{
223+
Set namespace = $Namespace
224+
Set SSLConfig = "GitHub"
225+
226+
Zn "%SYS"
227+
Do:'##class(Security.SSLConfigs).Exists(SSLConfig) ##class(Security.SSLConfigs).Create(SSLConfig)
228+
Zn namespace
229+
230+
Set req=##class(%Net.HttpRequest).%New()
231+
Set req.Https=1
232+
Set req.SSLConfiguration=SSLConfig
233+
Set req.Server="api.github.com"
234+
Do req.SetHeader("Accept","application/vnd.github.v3+json") // we want 3rd version of api
235+
236+
If ($d(Username) && $d(Password) && (Username'="") && (Password'="")) { // supply Username and Password, if both are provided. GitHub accept Basic Auth
237+
Set req.Username = Username // https://developer.github.com/v3/auth/
238+
Set req.Password = Password
239+
}
240+
241+
Return req
242+
}
243+
244+
}
245+

0 commit comments

Comments
 (0)