From ff43f097951c33beae51afabf94148e912b54b75 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Tue, 26 Sep 2023 21:18:36 -0400
Subject: [PATCH 01/15] add check for pandas NaN values
---
labelbase/metadata.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/labelbase/metadata.py b/labelbase/metadata.py
index 30da65a..d4d5f9c 100644
--- a/labelbase/metadata.py
+++ b/labelbase/metadata.py
@@ -3,6 +3,7 @@
from datetime import datetime
from dateutil import parser
import pytz
+import pandas
def get_metadata_schema_to_type(client:labelboxClient, lb_mdo=False, invert:bool=False):
""" Creates a dictionary where {key=metadata_schema_id: value=metadata_type}
@@ -121,6 +122,8 @@ def process_metadata_value(metadata_value, metadata_type:str, parent_name:str, m
return_value = None
if str(metadata_value) == "nan": # Catch NaN values
return_value = None
+ if pandas.isna(metadata_value): #Catch pandas df NaN values
+ return_value = None
# By metadata type
if metadata_type == "enum": # For enums, it must be a schema ID - if we can't match it, we have to skip it
name_key = f"{parent_name}{divider}{str(metadata_value)}"
From fbda4206279de8f5f9d8ff161d89b3f9191ee510 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Tue, 3 Oct 2023 10:32:18 -0700
Subject: [PATCH 02/15] deleted pycache files
---
.../__pycache__/__init__.cpython-311.pyc | Bin 249 -> 0 bytes
.../converters/__pycache__/coco.cpython-311.pyc | Bin 19341 -> 0 bytes
2 files changed, 0 insertions(+), 0 deletions(-)
delete mode 100644 labelbase/converters/__pycache__/__init__.cpython-311.pyc
delete mode 100644 labelbase/converters/__pycache__/coco.cpython-311.pyc
diff --git a/labelbase/converters/__pycache__/__init__.cpython-311.pyc b/labelbase/converters/__pycache__/__init__.cpython-311.pyc
deleted file mode 100644
index edb3e8503ee5b988395915ab6849de48972a4d7c..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 249
zcmZ3^%ge<81R`1iDT+Y)F^B^LOi;#WJ|JT{LkdF*V-7<UV+vz1gC^5Ukffg`(=ExI
z#H7@mq{QM>z2yA7vecrI)S_Y_GdcejCtPYJ!)K8CUm^OT#XxEOoYL&#l0<!%)Z*-t
z`~v-eqWrAX<dR~2=lr}pAR`~71ZFG}3t@qNe0*kJW=VX!UP0wA4x8Nkl+v73yCM#t
aC5%8^EDj_-Ff%eT-e6F;fQpLPfN}thT1Mso
diff --git a/labelbase/converters/__pycache__/coco.cpython-311.pyc b/labelbase/converters/__pycache__/coco.cpython-311.pyc
deleted file mode 100644
index 45df18a2ed1f3ce6bd2b55efda630b8382046134..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 19341
zcmeG^ZA=`=l|5ff55q7YFboXX3}B1_d+_>m7k{$GUgIyY@vd!b@9aP~FnE~3-Gedg
zOoDWq#K~DYV(rQ3;=P+hn+<!}NSu_D&?S-had+~cNZqI-S}h^1bc%F|?gx9D)$Qfi
zy;t2c(>;LK&PB=IX^Wof>iT$9uj<vSSMOEvBbTd`0$17A$Lu~vQU8V?IcL`bpWZi7
z)MJXF7*m`YHw~Cg7&gZ(<JJKy30vZ}0UN|w;w1wm@V3V7<MaTnr8x#15NC^*jyngO
zT3p$H3*t)R<>T%FH;J>yJ>#AM4++z8@3?QkH(oJNLB1XF%5ndIUn^TRP{ow80mjKz
zGi7XGxP)=NV;!g|iYte>U{RbK;%bZHJP=oBrdWDxTMlvZwLSO6)_(x_{}_I%W4s^v
zJ}^TJ{Ir+~XoZ=ne8>6$MEp1xO$~&K>heQf4f(h#wi?<VfVf6xI}>EhOfB2Q)Deuf
zy<;2L!C3mK*81NdyjGK9@8gnV>`)4Vp%FF}NhVUscyeSq!VitI<IzYe8A(LP*~mmR
zHF^m$**G@EO=F;EJUYT6gjc6iEZ@@$L2K$RGu~=ed{;*~Hp=uRlkwB{*`di)l2gi}
zd}Js&J`rbAEQ7`RlJV)0WI}N=(Nr|TC8r`WMk$RZ63J9F6~ly~Xo?+4a?_A7WXa=C
zQ6&(@?@u3q_n1mkX%l0r$!B29g>X@dWy=)nmMOME%7`^>28bnv7{>m@nz5v@OX+9@
zj9uup*gZB8iiYCRK{g&@Vnc{al$#Dk6AVNTCMOe2EHMIs`w*2wHZ;s7$3uKH#;65%
zho)ljcnCUsl1qeAqikp>Ih5QpluX=XIq1MG;S?$?4T0<8M)*z=!Y^G<hj#HPt`#G7
zu9Ji>V9T+2-7qkrE6J%)Pj_fI$w4z>yxP1xL>&gZu<Ferb`|g}v`<BubkofiQfJHU
zn=Q#fq(;8wHuS(~Y-kkl9%p$q?}SRKVfOw+l1oKMQGQozzA-v_c?NpjEFa^b2lcsC
z?9@(<6mTMtuaGg&^<ozc@1_XF)R8*Xd#V@vYdo4F02J#W^rm8sv%@LHl1fe}wy7AC
z8dXY0+1SWvN^uOcQ5a{ohiSDcbPVSekUZxBiss6}Q_L~uL+V2-SB5dJe6$;BLf2g0
z3Q0JGzwhgKoo6||BR+YDPenTb<vUPshc?&wj#J4*0=hYgS;%Ndc~%$7kr^ao72%0#
zC4iiO)Q8AGNF_v$tAje-@Z-n8o2Aw(DSt@z*3O@iyiHl~+7||vN~Ap}mTl6WGfzu}
z{u|QS0paGLbaqH!lfuM3iJe-fsA*G|`F9jGY<krUU*1z@InXGVSI^f<<<093TcBi}
z0&li=y^JcW6|BKep*CuBuJG@W0$jns6Ec<&Dr05L8C%-YP2IkdDM?$i2x5avmoPdm
zVPw)K>KUbU9OQXRF}4jkZA{4wjBv*GupjuEHEkmt$FAcz^b=dgp0=lL!)D+*j;Q51
zxK2ElVD&`%qTHsqd|?(73jd1B5XPx`Dii%Pa2XsV4#fw>bYC$|{|1A<K>>sFgZH70
ztHLzL9d>#m8B3&i#g&*GkK{iItKkBW2Uh2j=)?q@U|Q)Sp2O7>7=#gEeTolZV5k5H
z7Vcz{gGCOOZ=MT60iFgg*F{_{f++A4Qn<iR+A%Bq_`e4a_=%4ytC|~@oZDu*Wsh&R
zN3ID9)+#wryG~hcA>uu7%CvWtu3n*mUu65GnpTl+ljt^qZhMXduSnNQbge+w%5-o(
zD$?}=9farO%6i$qU9M@DtDEwi$XSA%2t43K<;aQZ!h$m_Si>C3d7Ja2>yYw?N-2zW
z#KNdNz$kUt@=}NGIp#kZ@c#`1*Z|-w4nXA6FO{5OBcsHHfZn(U^cvA?LJv6rw*$Ru
z^l&-JOAG3Rtj;g+1d48j4@2foEbtn5xfQ-DlnqE#tqV<3Rd}&<>9Dl-)bc55@7bpf
zg8u@{(PvH?YjERqK^8%Tdf4FMzR>gzF5x;+zMC@cgEE#BGB?eoEyksEET5LKreO~W
z&|u*=LKrcnt&DEjhA*A44_~?|XUsVR`&)?7Ez8ETm_A~CX8Vvk69-t*y5$|V7>whI
z9Rw8{Ebpb!63F-kdx{N{^F^2U33V%`XAob_?=Ri_9?Fw6(2Wg#74Z8MXXj&T1a`r<
zeHjxF6ICE4qYOM^{;v691rU{4$0Cp&z>eJ<-mBQivdtr5hC+lX{J!*PJjSPR+kSi|
ztYZ*l=U!kD;iK{7P&CdThd?g>>i~vt25*)Ue7)+kXq!7d-}LUetSvkAeyLE=o=+kC
zK(T9tXu#A8FbJl&)6g(=Af_P4TQlag`MZ{fwuhzt8f!y@KD2Nz!Ar68lY`hG#TrYo
z<2<q#kU|J*+<y3YQJii-nNKyGg1rKHvac;qsiCFR`~fwj`UEE;kf}Tvx$$qX4XQk5
zSLLxXo*fxy6ROUG?D-UH6tpKq5>xCkeh5?}3=z&+Eu0gID<wpUz;mElqgqRfBs9f|
zofsiJf#*(ObwmV1r3oMpPUZK(t%sBAg1i^t$NvC4B7;@*nP9Azg3CS9E0-W7`g$c_
z@9g<CSLNJI(bXW~vupOu$FTV$-LmjDbb=V{mEfzUPxe*Md*5xHe<R!V<i<kD!rhM?
zi#wORKW!Cv9a(x~x$EaQmhU}{{`0hWtWWS>Uh{$+CwiMDeA;rdmt5(GMP=PhqXY&X
zNMIlv;$Dz+^awN++?ywXnjI6c!H?kvU!fULL=j2VGRegagoQ&vR-6M6cueWJbz${L
zQ$_0vP7mT>BAI4>+mjeda-fbtSr^qcpy&bhS!^f-5_Xa(ZkXg$BA$#gq2zEV8XAxC
zcS0SZ9LMcR!2Ucj#D-qK(zD4{N~dQqs8`fIMwec_Gy!^{Xe@!MC8#*3e^ON?pzR<_
z0@1R-_I(SzMhIm2K2TI~Tyz>+OVm-=V3NPpqR<?Bcar5{>tF$0p<)NAJq4uOYT~+~
zOL*i@8g;pHSVyot*8?eom<6{Nv(#rEs;u(i(RqvLtdsEBAy{{86bT%VnGDbjbo#d(
z#vwBDHsc_HXn^H3H<ll=R8IK}DPzc2h)dz5(p<3^VT5b^V$6*B8}QS7%Qwu!I?X}G
znnF#D=Em|hr>!Ye<K*H|>><t&i<lGl0pRiDJT7`+1qB5C_jeqd<NUxeY+`Kh&?yuF
zHOE*}I1rlqw!?^*Q4L)zG#T?|N>YX8hDw(hQ^)c>02&KW%!3eCQVb7-FuNWP_|o+v
zeCdTOLrx-u+4aJcv8*P9J*$1FT_X;t$x+D2pb%CHLKvN<VVXOmZXlsA5W>jB*3H)!
zmBIAOfXXFwz`l~kzI4)<;*4p$8(8-xb%jSpV_0JHQrcUP^4BI!dTGm;z72K+9lhyf
zHog4zj9j|XM6PQmave56Z3>a=G%?5&Q%Y3XP7{G<bs|uEUJgJCezr_E4dD4h;C&84
zUG1Cdv;2~}tY_~&xwX*nB(gBPc=@B)lIf%P(n}vDg}USUoGORcOR>m<Ur@H=uzn2>
zy(sr0cw_(s_X>KbwNbapxWiSq$wwg6>LzQJVordzo-BwQ3Q&qQ32HG8nYm&ay0=d$
z8;&KIQ)o1s<aot#<@D7nkqbTjSCxuWXS;eXMY{U>uJm4y^!HrsyKoweXVJt6t61Uy
z0~Z}tJR@RUX|WjRAj5$c8^t{cx-^&oBnBFz(75nojoisR6cqBfOW^Ua_oRrRhu%q5
z+AFBrjHArmg3NI&bpXZ%hV?Ta<=s9%wy;gwepvLpEO}m@?OCg;es}b}$tN>EX<2L)
z+m1?YN5%SMQvETp>bO*OeCaO!Eqlq|zU8jveM|SGsxz~_Yi^(9uFtyvvErxO{>37l
zye7J@OYZB!jRDDhbM~z4^3O#d-kd$N=Bs>nSDs2B@Jrj%=&vqc6<)t7UcMz=z9pW$
zEuFm$31aP=68OG|<coZngiodJS$0!{S`b+R5o-PwCM)VWK#_+=MV+QY#~5@=A($_$
zX;T)8Z*)ez#yyJr8bH)(Hn@AnZ`eAGXKbKYv}HE}gP0BQ4(f#q5Il5x662De%8OML
zbqCoI#Z>`*w&%gmL?s#@UL?77nwPCiyIb`V*z@hr-A_KfNQj2ywACnbWl9nc8Tz^5
z`BQn(4KzfXz`F%~k}1hp5>dwS+)#Z65p@%&(8^bs(iec27tA(qCBXBh(5k1o%`gEI
zT1nbMj8M+#&j78^H4d!?U?+x+4;KR~CMcAc(D-CL6-$u=0eM5VUelzi{&}#HhxUYG
z!=VIhFwi_3kMcb1so49lf#0o`ESv{W&EtbCi64$}d@966nOL#|PAv@G!9BkUAU2HK
zK!hDl@KeyX=)GtROrCLelik)Ana7cz{_#!Z1*lz{T9t_)NvyPgBs2N5SXD^dW^S^<
zoEl|0HuN^wi;ijb>V9%Wq=$h{xfhL3vY}(4w;Ng55Hl<jAu)}aycM70SbidzV0a`C
zo$Y<Ay=Xz?g(CpD!rJaOrCRFsRK!n%3gtdf5IHNM5egn$nB;+&$YBbNTEGy#C(ho3
z!x<?!pAsVVt7bD=J@eATC@upQiklE)o|bqpj}dB8?CP6WZS`my<MyD38iiI`u|j9@
za4Kecf>o@D5V4kXXyW2fZ$}g@WC?Dy>(>9<kfiv=qxXvpYwBTua${pu?G8#LNJ!w+
z1%F#!tSf9I3c?*1gZ$rwAOBrg_KAik2%3~C*%w}zS!$CyPD{Qs%U_p#*JjVl_Uid_
zf7kmry<*)Sscw&8$47Shg{l_GxeHV=j-cGQTiA14YCN${*<C^6&6Ud)wew$Ft>3p&
zzfY_`Ak`lbD)3ng)Xra8cv-4DAO;RffrG!OTnQW#0>|W?uPz@KuH2GN-IjK~DMS*|
z&g8sPuHH6pTenfw$Jc49>BQ34p2nmTM1nYRQ@AxOofwfCM}^pZsd0Mlyj)o$+g)?P
z$MuiuMQ@|zZ4~VItOZ|MY+ssrdPEFflY-YCxF5J>hkLH`@rg$##EKnK#SX!N&szBO
z@*Sc7wh$Q;!*`_c9g%K-;1+7y;gKDlxpR+uAN7ir%~EBv;J{}sxNq^ua_Q6aV(_{Y
zysiTF&5b=yK1zzvJyk7&1E00vfyJB4ZBOrr!2u~afcbu1+$Px>G#V4H{@&8j(E&P9
z_0)@Ehm${pNuE{gOzd8aVL5|Xf#M_)D+Jin;^_x2pjT)yrZiPhYJHD-&um<_;ba4q
zCCfFyF*wHjQITl{1Qru8d7T^&`h>iG0~VB`{NICPdhZp*VSNP+3{zrk9b<pb!q6!c
zWHmR2ViR#$l*2bTqrjrmwouu;hDBefD1Uy@{=@6c>RMZ<i7C>pfL5Ud1UV;D20zym
z%ZMpsi-AA@J?b{}X!$1NqtjCrYA|BLoriIJj(i{h9>h~5S)DFwE0~aXJ7_{mvO1xm
zkT*>i=qgReOvUYfSpV=);vBRpuWKokEfzFMs~l;^pQcsN=N*PVcN?g<NS6gv?Ab`g
zhVtHG+~LeoDg2yJQ#OUGgXW572;lu!nKBTk${1fktSy9~_YEm&=RFfQnRaG9If(kV
zF>i63(xsc1V#;77y*W4#_T|D%MWH2z_)0@O?&FEuVkY>FEiux~SY@s>Q}x7d%xR(?
z&Ltj^b6a2alGaU|_tFL~5-9FTphnk*T-C+7QYfKl4rVT%8`|KEd}l1LPlolP#;8Hg
zlrzDM8*27I&7Y;vGN(BsjDu9h`vtS42WCkbQ(K@rrmhf%S@d$om-cK<-)V2QkTur;
zc&AWS)0}bUq<!h~4`4g{<NTJkxSt-r^6;F##i_$NNU#cejJb_p(AI02igZP`u27;7
zGWJWRGF{1(6&jy*-ytpHf+<v0X>KfkwA`56SZ~Iic5Tr=l}xZuhp`mQ!0Qf**@l0{
zv4t(Wvx#~M4wFoo?^|wAQzjF2gEGNiLV5N`lllB*9M5Nr;s5_pgBfaE6So`IM3<2+
zU`?$5f;BN@EMbC^3K@U8yqlUc-TrN+DqWtgDo`EMpbIx@>Z1>!L;!x(DKutljtQpS
zMoNUSxtys<2R5hfR9CJfMKMjrHtEySHB8wIWHsj47z6Ta_2xCDYjQ9~HptiyucZB3
zlrr|ijzT{a^-%McDf%*jbTzZ{$%fO#4}Vq!P2b-wT7P%{Y1+uNjF^}p6C5#b*3voC
zLvK-AcNH*aTD7*G_!G6Y?a7Ay3$$?1FgJ`N*j{LDwxK|Kg;2VPI*b&@?A9=R<2f;W
z;rjI4zNn|4i^iDE`{X}1NEdsIn8C~`-s^-n<iK<9rD<2%&vZP&V{m(;epE2#Ot9LP
zn5Y-wqyuXE;J5<Ok@4EinISkr18O4F29f(1>OqLm7@tgRqWQ}KMEwb>zOKMI2XaYd
z7Xb=KCc>(2D6HiS_i5LrK;v{FcV9+}3#oTlNHqvh+qYwo0Pme3SAyc9U4x*2Y6S>A
z!=bC(B)dB_!QsJdI6I<VA0c{6IP3$;Owg%;DirThL1Z4ZrU}>tG`ZcgC)5v$SU3^F
zuy_Op6rp%J1iAz@UmZq0QirA>#fx6#wof!PLas8Yh=VGP)Vkp|Ot+?B#Vbxzpyv&>
zVC!3Uhg#yXAvOUjTnv*-EhJP>N8u+bh+FhgAyk=d)c8f!QA7<b&!+UZYi1fwlj}f;
zno0qN%R;CEyb1@}>D+;o8K0UTFBXx@Xwd&NzFbk%O{!O0W*SFRsR_QbqXU%M;XD~Y
zsSb?>_>LJzM-oQ&UW}cZvAh)CJ3}|$=x84AXl7bV;9eUGXSZTxu*vxtqQOBuw_=@!
z^IdpMJ^{lss9ui))Yynhwakp=Qb!lD<HC`zZr~Z}VK6J+#G7d<;M=WT(9-TlY4jHk
zhiAM+NQOf*75V#Id`L}YnVI%eDz?~r1!&~#lGf!I3tf&tOEIk82jg(R#^IK4#@l>j
zPxJVmW+v2pwzK(SXLEn6U9oF}4|khlah6;mEEG{(3_&(Jfw!I*6*;hJ!eu5nqcyXu
z&=9S4puyd&Ni$(x>kG<roIhxS$W8q-<SG^*K*)fFQ&MrL6qQ|yjm%xeuH8DjG}Bf9
z6H<W_!^A0#R>E2&8YpHOb^7Nm>YO>OYauqM7g~nbD~p<_b#f@*RF1{|MU{bKBNrsN
zNsM?4yjGjy;wQn*l2f9S{!`PNu>j5AVd0i#GC_3Fpo9h!H7KEx*I2l(AsF1OkphY%
zf4P%b!Ds674cr9#nQFpHH%WmuS#jDcbZ*isCF+z{oVwYqlxl5`v3zTzVk7KMu}s3n
zPZz7d)W==Vw~$6!NyWy&Y@SF?aU5b;s?C$g;Iv|cTdJ`*Af4idNj5Bi6w50f4JJn5
z1yj)5Pfj2>#XcBKf$@UpP&vr8AWSEFKgDv1XdJJhddA`WU^Eg<sq>TN3spu_a10JE
zt0GVi_hAa1GpHz~2_&_6YzF8HRf<Xp96%k90T8SP6@*F&m_fiGrdWrPaH|eZC$bY5
z;;vz$1&~l|{6svKY7KICun^kjxp7Q%s@x510+8!3F!~*gb`-F)V-PS0uo}jSjUdW>
z19O(=Vk5*z284wR7xzsBMNAHrBk|-QbO2f&^4EeC`|u=AQ#kX--X9{j4;9m}Vu2X$
zTL{)&WDVkw2)VDLcM&`u4<?WWYEw@=iSixHjoy#2#C_;HVw0fV>y*jXWm=<MLivfs
zmY=r63m%a^DbXhd`lRfuS@pH7_*xb#1z(HkJ0$rIp~a#c%k`+`Ui;zx?Av0~eyM4{
zQ2v_wi1b;BJ}b~?Kd!ESQYM6BLjQo&|0V|1XDYqstpNkVs(0s#cc<vxCE>GYX=?Sv
z<&_hc#S{G!J~w_9Pf792e^GFQ{Z;dSs;^J*)s43q@kP=us%aPTZsR5Mbu-CwL(Ou-
zyl%6U!I@U@X1l@65v<Qf7yW|gFh1^=XU~32?mekU9U^^kK%@^!^g)3>2)M4=11t7`
zXs?m%HDG(O9hC#yR|738ffg~)Dg|2CDYN4!@gBS;*EOxy9asV5OWh%<?htg73zEQ_
zqh(LcswcGK31xS^zk9J-Xy_C@M<mY?fjsLL%xOVLG*z|japX}%^zW4XJ9W@c%XJN_
zb^BK8_K9@|z`*l@&_6~D&9>tN!_z-(%ErDQem^XfpH?4{J|od*1o{j~KPuA4B>I>@
zACq0~*)y^~FnfuNdxfoH&0jO$o@*WLs6ZQ4#e>swAe8lI(^A6$FgrasE4wS_Zb|OO
zg)nq7JhEr|s;7Cy)0|}&E=W6H5j~xfr&I8B%ATsnu1798)Ua4By>wc*G9X>KBf!94
znUuHhgmW@Wc4_-b;p#2v>d5NViIuApFv^_iG>-D?rZYIo_>yazo*Z4gE*(6xZnxI^
z;O1Jr?-R=Et@s^!@ae7ieBFV~t)!|NNSpr;HN2EkC+iV1I<JUyr$l!Obf@gA%umSN
zM9xp&wWd?cudQ}pUFp6mcE2Iv6BXhULepKb>8{jtSFncU+B(5nEth*{U(<%Sea%&`
zo;b(uY0vhr)oh!;Bi6J@HElpYE}UfO&)MY0mIeF!;l&!M@d(hO7gNxmJ11AxKECwm
zQg;83I)BjltMK`!uM7PV>Ec_$U>wFSX*!Q%x8nj19rzITU&Ju@kX-OB2<_MYn)h~R
zr^H}~6zmZFdnNx~7))0?@#ai&Rp9aHqtWk<y*oB%kt?d^?!J3SsN1_zu~(?ryVl<E
z(Xpk9rOQ9B65CHo?I(rGU9zWk)zi4*Y0U1=-hKa&=xLKYZ321L9S9d_1c2wfPcZbm
z{AVGdDD?T~pBKfD&(A-rq{>5v{*-HW{CWDZA=@JQ+ax~><NuldgF6UF_-5NVb+t+U
zV*k?H;*s;xk@G@%Qhh{vLZT-GdV<W2vm)Ii(LDm)BYP`m&&joi*Qt7&`xErQFzswv
za7)gYm#QS^$>lRE&U1qEoLn7z&?5(G9-LkC2j^cBw(k@D`z8N=9412ZZ9r!^_Wi?(
z?6vRLy<hi}p~Z&9u77CyaP*^5u>t7lutb&=*;D=4{m7m5X1lV{@1M%jqNi2zv<l>r
z%c@q(wyl(HlN*mOeO+qo75d(g8n4NL{foU)pnEwY1zwkH4lX98nlsDyq?$|e_Ljv_
zxS(GTyc@hvD4WauJM`c)=ao_AZ3`Wet8?j)<a%{^WW{w+a9w=ys_J*n#}@A{H7pKq
zSs|$({zrhJ!r>Ly1;KUUg)2lp4QPP}d>YXL4@nco3cSxssdBI2-LsICyvLW`mb~Xz
zy_Z+KmxU`gMDKv)9S~hNCD%<M^45y$Ey4BHCe`1R8gH#O4z4r~3PWRJ;~l9H)|GLo
zF|l=jZ`r2*UeB@M4nk&oL^>?dVSx^7;!rz@)?JPE*DTOWZwRlyCER^maHjFGrum&9
zf&H@XRC(WFi*VT5cf>9nH-RsDUp;WuXOVo?t5tTX#sq$=`6@RCxm%k-?ALT49G*c?
z+__I}N2>VL2oO%gIboT|&fF8o&iy4RGm=b>#My9kA_nJFIF_?v(iu#$^z~d&kKfwh
zh6)JFeHd$wCCNctBEu`bp=4rc63z)FQejPCB_g;gb$j^H=mZ;|4v(<#??m8|j(YsI
zAFKFV^oG$xTTId69K{9-4{loV#)cyx{<35@A0Y>N9H(>N<ft$?#zO>s_<tX;*COJq
zdfS#9GR9l)zo(CmCz;7Odz||x_{0+w{59B!f^cOr$yA46d}Qi`V0>iCIZOUz$~{Z|
zWU4_hKDs2oKp7sH@(H;|rm6(vBO5bA*^LcQ+apsu1mh!9rL*L3-Rd^gE?Cwncr97U
z>uD2t6?=`<rVh*qFU$xp%m}Yy?|=s&f`A0CrAqR83Y-tVioG3`kP(vLwFJT^yq@-x
zSFyJn_~xwtomwFfo|}{c;e}~2@QM_8Ww~20^@ye($<(uMwws#ghCrS+H5a^3ER^}2
zDVMzNt5$J^mx@BIiUPC@qIBMYdcYNc!E1>jFCog3cd_>>Tx>LDOzWT}h1WcE1_>+z
RJ|wUVQ1m&T^+is%{}1O%h^znr
From 889e4294f49931ff0c041af5cd56009d2658f50e Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Wed, 11 Oct 2023 09:55:30 -0700
Subject: [PATCH 03/15] improved upload result message
---
labelbase/uploader.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index a12ab38..0ad7002 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -94,7 +94,7 @@ def batch_create_data_rows(
"""
# Default error message
- e = "Success"
+ e = {}
# Vet all global keys
global_keys = list(upload_dict.keys()) # Get all global keys
if verbose:
@@ -103,11 +103,14 @@ def batch_create_data_rows(
gks = global_keys[i:] if i + batch_size >= len(global_keys) else global_keys[i:i+batch_size] # Batch of global keys to vet
existing_data_row_to_global_key = check_global_keys(client, gks) # Returns empty list if there are no duplicates
loop_counter = 0
+ if skip_duplicates:
+ e['skipped_global_keys'] = []
while existing_data_row_to_global_key:
if skip_duplicates: # Drop in-use global keys if we're skipping duplicates
if verbose:
print(f"Warning: Global keys in this upload are in use by active data rows, skipping the upload of data rows affected")
for gk in existing_data_row_to_global_key.values():
+ e['skipped_global_keys'].append(gk)
del upload_dict[gk]
break
else: # Create new suffix for taken global keys if we're not skipping duplicates
@@ -135,6 +138,7 @@ def batch_create_data_rows(
dataset_id_to_upload_list[dataset_id] = []
dataset_id_to_upload_list[dataset_id].append(data_row)
# Perform uploads grouped by dataset ID
+ e['upload_results'] = []
for dataset_id in dataset_id_to_upload_list:
dataset = client.get_dataset(dataset_id)
upload_list = dataset_id_to_upload_list[dataset_id]
@@ -149,10 +153,11 @@ def batch_create_data_rows(
task = dataset.create_data_rows(batch)
task.wait_till_done()
errors = task.errors
+ e['upload_results'].append(task.result)
if errors:
if verbose:
print(f'Error: Upload batch number {batch_number} unsuccessful')
- e = errors
+ e['errors'] = errors
break
else:
if verbose:
From 53507c5e1b69c78c0b60d45904b22c300b94294a Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Fri, 13 Oct 2023 10:50:03 -0700
Subject: [PATCH 04/15] improved upload result message
---
labelbase/uploader.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index 0ad7002..85d3743 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -153,6 +153,7 @@ def batch_create_data_rows(
task = dataset.create_data_rows(batch)
task.wait_till_done()
errors = task.errors
+ print(task.result_url)
e['upload_results'].append(task.result)
if errors:
if verbose:
From f7c6cb7813cfc689fe71e86c6e56b7e2716ad564 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Fri, 13 Oct 2023 11:31:23 -0700
Subject: [PATCH 05/15] improved upload result message
---
labelbase/uploader.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index 85d3743..489ffe4 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -153,8 +153,7 @@ def batch_create_data_rows(
task = dataset.create_data_rows(batch)
task.wait_till_done()
errors = task.errors
- print(task.result_url)
- e['upload_results'].append(task.result)
+ e['upload_results'].append(task.uid)
if errors:
if verbose:
print(f'Error: Upload batch number {batch_number} unsuccessful')
From 54e41da66ca454c080f8cedb39234b2ff8e9837f Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Fri, 13 Oct 2023 12:39:26 -0700
Subject: [PATCH 06/15] improved upload result message
---
labelbase/uploader.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index 489ffe4..f709989 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -151,6 +151,7 @@ def batch_create_data_rows(
if verbose:
print(f'Batch #{batch_number}: {len(batch)} data rows')
task = dataset.create_data_rows(batch)
+ print(task.uid)
task.wait_till_done()
errors = task.errors
e['upload_results'].append(task.uid)
From 02fee1825caa47ed5e3c16f754ace0b32a581488 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Fri, 13 Oct 2023 15:12:55 -0700
Subject: [PATCH 07/15] improved upload result message
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 0bbb40e..f77f44b 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@
setuptools.setup(
name='labelbase',
- version='0.1.00',
+ version='0.1.06',
author='Labelbox',
author_email='raphael@labelbox.com',
description='Labelbox Helper Library',
From 33f9959dd8bf0f6b1c7bbd10cb79df08312a6d64 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Fri, 13 Oct 2023 15:57:56 -0700
Subject: [PATCH 08/15] improved upload result message
---
labelbase/uploader.py | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index f709989..47bde2f 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -37,7 +37,7 @@ def create_global_key_to_data_row_id_dict(client:labelboxClient, global_keys:lis
global_key_to_data_row_dict[gks[i]] = res['results'][i]
return global_key_to_data_row_dict
-def check_global_keys(client:labelboxClient, global_keys:list, batch_size=1000):
+def check_global_keys(client:labelboxClient, global_keys:list):
""" Checks if data rows exist for a set of global keys - if data rows exist, returns as dictionary { key=data_row_id : value=global_key }
Args:
client : Required (labelbox.client.Client) - Labelbox Client object
@@ -52,20 +52,18 @@ def check_global_keys(client:labelboxClient, global_keys:list, batch_size=1000):
# Enforce global keys as strings
global_keys_list = [str(x) for x in global_keys]
# Batch global key checks
- for i in range(0, len(global_keys_list), batch_size):
- batch_gks = global_keys_list[i:] if i + batch_size >= len(global_keys_list) else global_keys_list[i:i+batch_size]
- # Get the datarow ids
- res = client.get_data_row_ids_for_global_keys(batch_gks)
- # Check query job results for fetched data rows
- for i in range(0, len(res["results"])):
- data_row_id = res["results"][i]
- if data_row_id:
- existing_drid_to_gk[data_row_id] = batch_gks[i]
+ # Get the datarow ids
+ res = client.get_data_row_ids_for_global_keys(global_keys_list)
+ # Check query job results for fetched data rows
+ for i in range(0, len(res["results"])):
+ data_row_id = res["results"][i]
+ if data_row_id:
+ existing_drid_to_gk[data_row_id] = global_keys_list[i]
return existing_drid_to_gk
def batch_create_data_rows(
client:labelboxClient, upload_dict:dict, skip_duplicates:bool=True,
- divider:str="___", batch_size:int=20000, verbose:bool=False):
+ divider:str="___", batch_size:int=100000, verbose:bool=False):
""" Uploads data rows, skipping duplicate global keys or auto-generating new unique ones.
upload_dict must be in the following format:
From 8f1c2cbb19d8cc6f348b82fc2c7f5990e508748a Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Mon, 16 Oct 2023 11:53:07 -0700
Subject: [PATCH 09/15] improved upload result message
---
labelbase/uploader.py | 40 +++++++++++++++++++++++++++-------------
1 file changed, 27 insertions(+), 13 deletions(-)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index 47bde2f..5f56896 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -2,6 +2,7 @@
from labelbox import Dataset as labelboxDataset
from labelbox import Project as labelboxProject
import uuid
+from concurrent.futures import ThreadPoolExecutor, as_completed
def create_global_key_to_label_id_dict(client:labelboxClient, project_id:str, global_keys:list):
""" Creates a dictionary where { key=global_key : value=label_id } by exporting labels from a project
@@ -63,7 +64,7 @@ def check_global_keys(client:labelboxClient, global_keys:list):
def batch_create_data_rows(
client:labelboxClient, upload_dict:dict, skip_duplicates:bool=True,
- divider:str="___", batch_size:int=100000, verbose:bool=False):
+ divider:str="___", batch_size:int=20000, verbose:bool=False):
""" Uploads data rows, skipping duplicate global keys or auto-generating new unique ones.
upload_dict must be in the following format:
@@ -137,7 +138,9 @@ def batch_create_data_rows(
dataset_id_to_upload_list[dataset_id].append(data_row)
# Perform uploads grouped by dataset ID
e['upload_results'] = []
+ e['errors'] = []
for dataset_id in dataset_id_to_upload_list:
+ task_list = []
dataset = client.get_dataset(dataset_id)
upload_list = dataset_id_to_upload_list[dataset_id]
if verbose:
@@ -149,18 +152,25 @@ def batch_create_data_rows(
if verbose:
print(f'Batch #{batch_number}: {len(batch)} data rows')
task = dataset.create_data_rows(batch)
- print(task.uid)
- task.wait_till_done()
- errors = task.errors
- e['upload_results'].append(task.uid)
- if errors:
- if verbose:
- print(f'Error: Upload batch number {batch_number} unsuccessful')
- e['errors'] = errors
- break
- else:
- if verbose:
- print(f'Success: Upload batch number {batch_number} successful')
+ task_list.append(task)
+ # task.wait_till_done()
+ # errors = task.errors
+ # e['upload_results'].append(task.uid)
+ # if errors:
+ # if verbose:
+ # print(f'Error: Upload batch number {batch_number} unsuccessful')
+ # e['errors'] = errors
+ # break
+ # else:
+ # if verbose:
+ # print(f'Success: Upload batch number {batch_number} successful')
+ with ThreadPoolExecutor() as exc:
+ futures = [exc.submit(get_results_from_task, x) for x in task_list]
+ for future in as_completed(futures):
+ errors, results = future.result()
+ if errors:
+ e['errors'] += errors
+ e['upload_results'] += results
if verbose:
print(f'Upload complete - all data rows uploaded')
return e, upload_dict
@@ -491,3 +501,7 @@ def batch_upload_predictions(
except Exception as error:
e = error
return e
+
+def get_results_from_task(task):
+ task.wait_till_done()
+ return task.errors, task.result
From 1cbc5d09e3da35bf8733dbd230372eb337c20235 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Mon, 16 Oct 2023 12:52:34 -0700
Subject: [PATCH 10/15] improved upload result message
---
labelbase/uploader.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index 5f56896..9bf60f2 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -64,7 +64,7 @@ def check_global_keys(client:labelboxClient, global_keys:list):
def batch_create_data_rows(
client:labelboxClient, upload_dict:dict, skip_duplicates:bool=True,
- divider:str="___", batch_size:int=20000, verbose:bool=False):
+ divider:str="___", batch_size:int=100000, verbose:bool=False):
""" Uploads data rows, skipping duplicate global keys or auto-generating new unique ones.
upload_dict must be in the following format:
@@ -503,5 +503,5 @@ def batch_upload_predictions(
return e
def get_results_from_task(task):
- task.wait_till_done()
+ task.wait_till_done(120)
return task.errors, task.result
From b9b3682ee7207a5707aa232df32772eb4dd630d4 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Mon, 16 Oct 2023 12:56:43 -0700
Subject: [PATCH 11/15] improved upload result message
---
labelbase/uploader.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index 9bf60f2..7be2031 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -137,7 +137,6 @@ def batch_create_data_rows(
dataset_id_to_upload_list[dataset_id] = []
dataset_id_to_upload_list[dataset_id].append(data_row)
# Perform uploads grouped by dataset ID
- e['upload_results'] = []
e['errors'] = []
for dataset_id in dataset_id_to_upload_list:
task_list = []
@@ -170,7 +169,6 @@ def batch_create_data_rows(
errors, results = future.result()
if errors:
e['errors'] += errors
- e['upload_results'] += results
if verbose:
print(f'Upload complete - all data rows uploaded')
return e, upload_dict
@@ -503,5 +501,5 @@ def batch_upload_predictions(
return e
def get_results_from_task(task):
- task.wait_till_done(120)
- return task.errors, task.result
+ task.wait_till_done()
+ return task.errors
From 82304704f070d494c4d584b6e361a018ce30796c Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Mon, 16 Oct 2023 13:14:10 -0700
Subject: [PATCH 12/15] improved upload result message
---
labelbase/uploader.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/labelbase/uploader.py b/labelbase/uploader.py
index 7be2031..57a193c 100644
--- a/labelbase/uploader.py
+++ b/labelbase/uploader.py
@@ -166,7 +166,7 @@ def batch_create_data_rows(
with ThreadPoolExecutor() as exc:
futures = [exc.submit(get_results_from_task, x) for x in task_list]
for future in as_completed(futures):
- errors, results = future.result()
+ errors = future.result()
if errors:
e['errors'] += errors
if verbose:
From bcbd4cd495196c0e9407596b36a58a2297abd89d Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Mon, 20 Nov 2023 17:15:53 -0700
Subject: [PATCH 13/15] debugging
---
labelbase/connector.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/labelbase/connector.py b/labelbase/connector.py
index bc4c1e7..918440d 100644
--- a/labelbase/connector.py
+++ b/labelbase/connector.py
@@ -187,6 +187,8 @@ def determine_actions(
attachments_action = True if attachment_index and not create_action else False
# Determine if we're batching data rows
batch_action = False if (project_id == project_id_col == "") else True
+ print(project_id)
+ print(project_id_col)
# Determine the upload_method if we're batching to projects
annotate_action = upload_method if (upload_method in ["mal", "import", "ground-truth"]) and annotation_index and batch_action else ""
# "ground-truth" defaults to "import" if no model informtion is given
From 979ad1a086fd8f296ed4746474507a93d76f1f28 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Tue, 23 Jan 2024 18:21:10 -0700
Subject: [PATCH 14/15] include label details to export and flatten labels
---
labelbase/downloader.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/labelbase/downloader.py b/labelbase/downloader.py
index 65c8712..eeb12ae 100644
--- a/labelbase/downloader.py
+++ b/labelbase/downloader.py
@@ -5,7 +5,7 @@
from labelbase.annotate import flatten_label
def export_and_flatten_labels(client:labelboxClient, project, include_metadata:bool=True, include_performance:bool=True,
- include_agreement:bool=False, verbose:bool=False, mask_method:str="png", divider="///", export_filters:dict=None):
+ include_agreement:bool=False, include_label_details:bool=False, verbose:bool=False, mask_method:str="png", divider="///", export_filters:dict=None):
""" Exports and flattens labels from a Labelbox Project
Args:
client: : Required (labelbox.Client) - Labelbox Client object
@@ -82,6 +82,7 @@ def export_and_flatten_labels(client:labelboxClient, project, include_metadata:b
flat_label["seconds_to_create"] = nested_label['performance_details']['seconds_to_create']
flat_label["seconds_to_review"] = nested_label['performance_details']['seconds_to_review']
flat_label["seconds_to_label"] = nested_label['performance_details']['seconds_to_create'] - nested_label['performance_details']['seconds_to_review']
+ if include_metadata:
for metadata in label['metadata_fields']:
try:
if metadata['value'] in metadata_schema_to_name_key.keys():
@@ -115,6 +116,10 @@ def export_and_flatten_labels(client:labelboxClient, project, include_metadata:b
metadata_value = metadata['value']
if field_name != "lb_integration_source":
flat_label[f'metadata{divider}{metadata_type}{divider}{field_name}'] = metadata_value
+ if include_label_details:
+ flat_label["created_by"] = nested_label['label_details']["created_by"]
+ flat_label["updated_at"] = nested_label['label_details']["updated_at"]
+ flat_label["created_at"] = nested_label['label_details']["created_at"]
flattened_labels.append(flat_label)
if verbose:
print(f"Labels flattened")
From 3994764bcdae69fd0813b71449ad51b1f0b96db5 Mon Sep 17 00:00:00 2001
From: Luke Moehlenbrock <luksta@Lukes-MacBook-Pro.local>
Date: Wed, 24 Jan 2024 18:59:21 -0700
Subject: [PATCH 15/15] removed debugging print statement
---
labelbase/annotate.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/labelbase/annotate.py b/labelbase/annotate.py
index 8627974..b624923 100644
--- a/labelbase/annotate.py
+++ b/labelbase/annotate.py
@@ -365,7 +365,6 @@ def flatten_label(client:labelboxClient, label_dict:dict, ontology_index:dict, d
if column_name not in flat_label.keys():
flat_label[column_name] = []
if "bounding_box" in obj.keys():
- print(obj)
annotation_value = [obj["bounding_box"]["top"], obj["bounding_box"]["left"], obj["bounding_box"]["height"], obj["bounding_box"]["width"]]
if "page_number" in obj.keys():
annotation_value.append(obj["page_number"])