From 99ac47cf0df1fb43677790bd4b9e1260238fbff0 Mon Sep 17 00:00:00 2001 From: Jim Paris Date: Fri, 7 Aug 2020 00:11:45 -0400 Subject: [PATCH] Start implementing fsck test and porting fsck to Python 3 --- nilmdb/fsck/fsck.py | 7 +- tests/fsck-data/test1/data.sql | Bin 0 -> 24576 bytes tests/fsck-data/test1b/data.sql | Bin 0 -> 24576 bytes tests/fsck-data/test1c/data.sql | Bin 0 -> 24576 bytes tests/fsck-data/test2/data.sql | Bin 0 -> 24576 bytes tests/fsck-data/test2/data/a/b/0000/0000 | Bin 0 -> 2295 bytes tests/fsck-data/test2/data/a/b/_format | Bin 0 -> 99 bytes tests/fsck-data/test2a/data.sql | Bin 0 -> 32768 bytes tests/fsck-data/test2a/data/a/b/0000/0000 | Bin 0 -> 2295 bytes tests/fsck-data/test2a/data/a/b/_format | Bin 0 -> 99 bytes tests/fsck-data/test2b/data.sql | Bin 0 -> 24576 bytes tests/fsck-data/test2b/data/a/b/0000/0000 | Bin 0 -> 2295 bytes tests/fsck-data/test2b/data/a/b/_format | Bin 0 -> 99 bytes tests/fsck-data/test2c/data.sql | Bin 0 -> 24576 bytes tests/fsck-data/test2c/data/a/b/0000/0000 | Bin 0 -> 2295 bytes tests/fsck-data/test2c/data/a/b/_format | Bin 0 -> 99 bytes tests/fsck-data/test2d/data.sql | Bin 0 -> 24576 bytes tests/fsck-data/test2d/data/a/b/0000/0000 | Bin 0 -> 2295 bytes tests/fsck-data/test2d/data/a/b/_format | Bin 0 -> 99 bytes tests/test_fsck.py | 118 ++++++++++++++++++++++ 20 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 tests/fsck-data/test1/data.sql create mode 100644 tests/fsck-data/test1b/data.sql create mode 100644 tests/fsck-data/test1c/data.sql create mode 100644 tests/fsck-data/test2/data.sql create mode 100644 tests/fsck-data/test2/data/a/b/0000/0000 create mode 100644 tests/fsck-data/test2/data/a/b/_format create mode 100644 tests/fsck-data/test2a/data.sql create mode 100644 tests/fsck-data/test2a/data/a/b/0000/0000 create mode 100644 tests/fsck-data/test2a/data/a/b/_format create mode 100644 tests/fsck-data/test2b/data.sql create mode 100644 tests/fsck-data/test2b/data/a/b/0000/0000 create mode 100644 tests/fsck-data/test2b/data/a/b/_format create mode 100644 tests/fsck-data/test2c/data.sql create mode 100644 tests/fsck-data/test2c/data/a/b/0000/0000 create mode 100644 tests/fsck-data/test2c/data/a/b/_format create mode 100644 tests/fsck-data/test2d/data.sql create mode 100644 tests/fsck-data/test2d/data/a/b/0000/0000 create mode 100644 tests/fsck-data/test2d/data/a/b/_format create mode 100644 tests/test_fsck.py diff --git a/nilmdb/fsck/fsck.py b/nilmdb/fsck/fsck.py index a40c327..8b38a1b 100644 --- a/nilmdb/fsck/fsck.py +++ b/nilmdb/fsck/fsck.py @@ -211,6 +211,7 @@ class Fsck(object): # must exist in bulkdata bulk = self.bulkpath + path + bulk = bulk.encode('utf-8') if not os.path.isdir(bulk): raise FsckError("%s: missing bulkdata dir", path) if not nilmdb.server.bulkdata.Table.exists(bulk): @@ -256,7 +257,7 @@ class Fsck(object): @retry_if_raised(RetryFsck) def check_bulkdata(self, sid, path, bulk): - with open(os.path.join(bulk, "_format"), "rb") as f: + with open(os.path.join(bulk, b"_format"), "rb") as f: fmt = pickle.load(f) if fmt["version"] != 3: raise FsckError("%s: bad or unsupported bulkdata version %d", @@ -278,7 +279,7 @@ class Fsck(object): rkt.close() # Find all directories - regex = re.compile("^[0-9a-f]{4,}$") + regex = re.compile(b"^[0-9a-f]{4,}$") subdirs = sorted(filter(regex.search, os.listdir(bulk)), key=lambda x: int(x, 16), reverse=True) for subdir in subdirs: @@ -337,6 +338,7 @@ class Fsck(object): for sid in self.stream_interval: try: bulk = self.bulkpath + self.stream_path[sid] + bulk = bulk.encode('utf-8') tab = nilmdb.server.bulkdata.Table(bulk) def update(x): @@ -422,6 +424,7 @@ class Fsck(object): for sid in self.stream_interval: try: bulk = self.bulkpath + self.stream_path[sid] + bulk = bulk.encode('utf-8') tab = nilmdb.server.bulkdata.Table(bulk) def update(x): diff --git a/tests/fsck-data/test1/data.sql b/tests/fsck-data/test1/data.sql new file mode 100644 index 0000000000000000000000000000000000000000..aa1759e6b9b23a9ab7e7d9f5baa37ffd7783b087 GIT binary patch literal 24576 zcmeI%&u-H&9Ki9U>mN2$wp&$IlPq!QCbgYl7j95kgepqLN-J%r$f)k9)+VKKfgP*8 z1MkEm@Bq96Cq(|Wr2;B(X)a$&i5=UC^ZVF2Sb2NV9%>~9-gGRrI5iH8OvZR7gkc!@ z9^wk4wA8Pq-q{d3izSMG&=U;TJX4ewk=JU2C7PUuvB1yiUDmiwBebKVJ z)=R4+?DMX$d+l~@+2=;ximr7TRR7WRQI2LRYK(dx53+?)wQSDIL$|L!J5%XiE8mGO z;&Lu|pO$@QU5d31;@lQ-CvmiVt(x$)oN7l8$Eqfj+mA}e^ZrhuR4$u$52L8#+v0O} zE#kQD|KeSZxB+GHDjT}Q&zX4shCbnA{?;SroJ+G%s#MIoaujB);wM>VEzqQL73ptN z`li!*-RxY6SJqW+ceh>-Ov70ht+VAEPo#byPjb(0UG%KgT!;O`)hf%PrcjQca z@d`@9avmjxaNi#7!i(V-caYj=$G`|6fB*srAbz^+ s5I_I{1Q0*~0R#|0AUy&8|EFiq$Os^S00IagfB*srAb7su7)G5`Po literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test1b/data.sql b/tests/fsck-data/test1b/data.sql new file mode 100644 index 0000000000000000000000000000000000000000..aa1759e6b9b23a9ab7e7d9f5baa37ffd7783b087 GIT binary patch literal 24576 zcmeI%&u-H&9Ki9U>mN2$wp&$IlPq!QCbgYl7j95kgepqLN-J%r$f)k9)+VKKfgP*8 z1MkEm@Bq96Cq(|Wr2;B(X)a$&i5=UC^ZVF2Sb2NV9%>~9-gGRrI5iH8OvZR7gkc!@ z9^wk4wA8Pq-q{d3izSMG&=U;TJX4ewk=JU2C7PUuvB1yiUDmiwBebKVJ z)=R4+?DMX$d+l~@+2=;ximr7TRR7WRQI2LRYK(dx53+?)wQSDIL$|L!J5%XiE8mGO z;&Lu|pO$@QU5d31;@lQ-CvmiVt(x$)oN7l8$Eqfj+mA}e^ZrhuR4$u$52L8#+v0O} zE#kQD|KeSZxB+GHDjT}Q&zX4shCbnA{?;SroJ+G%s#MIoaujB);wM>VEzqQL73ptN z`li!*-RxY6SJqW+ceh>-Ov70ht+VAEPo#byPjb(0UG%KgT!;O`)hf%PrcjQca z@d`@9avmjxaNi#7!i(V-caYj=$G`|6fB*srAbz^+ s5I_I{1Q0*~0R#|0AUy&8|EFiq$Os^S00IagfB*srAb7su7)G5`Po literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test1c/data.sql b/tests/fsck-data/test1c/data.sql new file mode 100644 index 0000000000000000000000000000000000000000..36fdf42e4143041eb97249b9a9661dad7ff64719 GIT binary patch literal 24576 zcmeI%&yLbS9Ki9Gg{`{=cW;D{O(yZcZUSs{FWwa0#t^aVA|$(~W(>nhgcj+b?vd;} z_)b2858yj^GWH)J(U5qdm+vH{L+982`uoiE(DdzPd#IHd_|vh};>N(Y9#jqO)Od8I7mkiw8fALh+vQ;K#wY;;-V@uuDS#0R#|0009ILKmY** zG835JFIc6cBXd5~^4&;{m6m;}zlJ`^(M-jIarDU`Zk#NqXX_YD!^R671nWzNmF1J=_dT>?gA5;3K z(|XexFQo&uLxu?A2UH@zH3g-?(Xr>ZzwbYu`Ja zOx2{iJbxr_{h3~0CUL5CzzbJodWFHrm-_6n`y{Ecp4%-oE2a@IlHlVge5-fmO#8_S zO3QK|rJbGdL5ttQi_sHznAu}TVgwLC009ILKmY**5I_I{1Q6JQzqWs0h5lBFHGN&Ta-%WhCugepqLN-J%r$Wy(oTAGyBW$YO3 z9e5d@fJfi~I5X`qaRQt;ZK;4toRHeDmBfzi{P}(COX4?2&7R=&%pdldpnK#INhHWq zN(mvEc$Zm}Qt}w?lcdQ)a&q#QJVUBaKIe^ZB%RNZ^)JS!{P+9^Sww;W1Rwwb2tWV= z5P$##An@M=f(PlWxw)wYJ;6?0-sggKnP8JV^Sou%+ZJutpEWHysoh?qaTxl}G;Y<&#K54HY(PmKGOMnUipfB*y_009U< z00Izz00bZafh82s)AAR99{vGP>;JdJxLv|95fTI-009U<00Izz00bZa0SG`~MFrNi Qlx~_A+;#mcf9Q690v7T6@Bjb+ literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2/data/a/b/0000/0000 b/tests/fsck-data/test2/data/a/b/0000/0000 new file mode 100644 index 0000000000000000000000000000000000000000..c0605f5fadd20232bdca3ce91ff753274991eff4 GIT binary patch literal 2295 zcmXBWd0Y<$7{~D^6iSY?T7(=Ciq&$q@;D<$h$cy{oH-hiFb{G>h~_>bM-nZOhPdPOP+!Zp0k+g75oo}R(t(C@ zAq!}v#`gd_>irUE%qu?vP2`$MB|}(Kt!V``qe1q7J;%5L%_X-x&_bNexTrofHNl*11|FR3-F`5m{vA~xl*VN z(2mm_fcEma1JFU;Jb{k%t1r-rH3;Y|l|lhG?GXX|M9X6Ucg~6fJj5&o=%W440Zh@^ zKv%x~1n4G~B|vxetpIw^7V|2Gu%7(D7U(4n905-abOU-*oEOlC3;X~tX+8<)s}sY3 zew4Tz=+AF90t2M&e!yEpQh|YV>>@CTKimNZOUHb`N5e~jA(UZcZ0H%v-)aNC($x<5 zS(mg0hEb*m;K$Vl0K>&A0PxqSU|V^bh6x}@mjOKb*fH5*6 z4;ZUE3xRR;>^%^~O^r@~kc)}`u(p3UT23)sS&8v8k{1a|Ym%|M*IjtBN=n>1iAg=7Nz_-HP$ zU)~o0@!FvbI6&c*O$}iQeA)syC>4!?L}h2-5G`TgFkkWplB8-N@VoYz1|(C|0^kVW zhyqfi_IBW?4o(D)(S~#&mGiQI<6?6UNK^lpzzN#<5je?DP0S2o>C(gsIHlw4fzuT4 z3S{uB?!Xyo?E{?Esbc`8Hu74^Nzp`@%996>N;N_n{tAHTWl2y+?HVxz#ZKh z1LV-7I3SlBr2v1*=yO1x?#Tx3QsEQe9y^o(_hoVg@IViln;XI&Qi(0_h@BmQ$1>9m zc%rFZz*G9<2R!4>lYr+kKMcs%Gs}SjGTH?E#ohM5lxBmdvAK@(k literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2/data/a/b/_format b/tests/fsck-data/test2/data/a/b/_format new file mode 100644 index 0000000000000000000000000000000000000000..ca49cc290f2b7e665cd978ca27ec8dd76a3a715c GIT binary patch literal 99 zcmZo*sx4&Dh~Q;lU?|EjFODxrEs9Ue%t_ARwW?qR!ykQ{=m{*osRGgWgSIFwkT&f2Eujm|S literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2a/data.sql b/tests/fsck-data/test2a/data.sql new file mode 100644 index 0000000000000000000000000000000000000000..ecee5ecac080bc858ac8524650ceff95c819e638 GIT binary patch literal 32768 zcmeI&yKmD#9Ki9j^KcxR8VOZa6jeI41FaOODi$^G^ z-@w+tgg*qau(80!iWVsFpKreIs8b#Osx3@>F2dwyOUHUoFJ<+uI7 z_1wT6EwgvGoVA)GYioBZjvPgo@-iv?py#{oM$?n))tYnP*^eQ7&+R<+`;BNLUe6@gvtGUDY|F6@a-%Ba zPIBq&vdXd_xV@keG~0ez`W-J?TON8T!!8!Jmy1!>@ow=vJ(h7C`?Gw1o^b-o=u#$h ziH~#G?N8_v9_DvFVor4r9bV9V6rJXpm9^JOnurFh|D>g!up90m=*FWN4@$qWnylnr z%o=v7qzxZLgOEg%g)ue^3rTcAF6GUgrnh4zw_90l9=gH)jyZOR$$E8tv+jhgE%&H9 z2+pAbzHk))%2B z3IYfqfB*srAbYpxR9L6h$Eiw@WDr0A0R#|0009IL zKmY**5cppOvO>&iY16j%{Z^~{tlMjO;q$+4y%*Mrno$ry009ILKmY**5I_I{1Q0;r z9~7|kv^iH=d9>mUnw{WgW7SfR|0>|F>?VQm|NpvmBCM}!MnM1p1Q0*~0R#|0009IL XKmdVh5XhzGwD5I+ZlvbS@HM~>Eh~_>bM-nZOhPdPOP+!Zp0k+g75oo}R(t(C@ zAq!}v#`gd_>irUE%qu?vP2`$MB|}(Kt!V``qe1q7J;%5L%_X-x&_bNexTrofHNl*11|FR3-F`5m{vA~xl*VN z(2mm_fcEma1JFU;Jb{k%t1r-rH3;Y|l|lhG?GXX|M9X6Ucg~6fJj5&o=%W440Zh@^ zKv%x~1n4G~B|vxetpIw^7V|2Gu%7(D7U(4n905-abOU-*oEOlC3;X~tX+8<)s}sY3 zew4Tz=+AF90t2M&e!yEpQh|YV>>@CTKimNZOUHb`N5e~jA(UZcZ0H%v-)aNC($x<5 zS(mg0hEb*m;K$Vl0K>&A0PxqSU|V^bh6x}@mjOKb*fH5*6 z4;ZUE3xRR;>^%^~O^r@~kc)}`u(p3UT23)sS&8v8k{1a|Ym%|M*IjtBN=n>1iAg=7Nz_-HP$ zU)~o0@!FvbI6&c*O$}iQeA)syC>4!?L}h2-5G`TgFkkWplB8-N@VoYz1|(C|0^kVW zhyqfi_IBW?4o(D)(S~#&mGiQI<6?6UNK^lpzzN#<5je?DP0S2o>C(gsIHlw4fzuT4 z3S{uB?!Xyo?E{?Esbc`8Hu74^Nzp`@%996>N;N_n{tAHTWl2y+?HVxz#ZKh z1LV-7I3SlBr2v1*=yO1x?#Tx3QsEQe9y^o(_hoVg@IViln;XI&Qi(0_h@BmQ$1>9m zc%rFZz*G9<2R!4>lYr+kKMcs%Gs}SjGTH?E#ohM5lxBmdvAK@(k literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2a/data/a/b/_format b/tests/fsck-data/test2a/data/a/b/_format new file mode 100644 index 0000000000000000000000000000000000000000..ca49cc290f2b7e665cd978ca27ec8dd76a3a715c GIT binary patch literal 99 zcmZo*sx4&Dh~Q;lU?|EjFODxrEs9Ue%t_ARwW?qR!ykQ{=m{*osRGgWgSIFwkT&f2Eujm|S literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2b/data.sql b/tests/fsck-data/test2b/data.sql new file mode 100644 index 0000000000000000000000000000000000000000..0af2acc780449d00f9dd7ba1ccf425a6ca38c33f GIT binary patch literal 24576 zcmeI(OK#IZ7zgkf=i#`JnoVUzQKgZ(NGs)`(oJ^|sD&azN<$*GD{HDuTMNey$%N8n z)H`q)PQVd301Gxyu|TYV@goTl5Q!Bc`hRjf!uO*(NUKlMakNZ#X;Ps{AXwq5VotW$r`wAr+EdyOSAl0(7! zj@x66R>yv1w^{43!&=>DvpnnbTwJk^eXN>)+4YipV-f9)dheX2b*ogMlY;B_#4BgW z{c|yLqK&woN$#i7IIYCDr&O1BCHdc758 z9Um62(+e5LZGV^V{EACZrnj=BOMIR|Fj~?lJk9TV#GDgOb*osU*M%s}SY?zH+Crj9 z<2=)!hV*H>@uc29VUO*TvSCyz$}}8x);O4r@qo*Vc#ylT#!=Uvk9F8T+^h7f=T%m! z`QlOrer+%mgVOAHk6#62x%ikwmr8c{>WNIwaOMSE?%j3v;~Xnm)~XjNi5`;C3s1e( zJA5pI_z6nVa(0tS_}(79h38d;nxBYT5D1Rwwb2tWV=5P$##AOHaftcHL|(`2Jq z&ty-cTMF*)%SU|ARqt%2tWV=5P$##AOHafKmY;|STO-TwPA$Q0O9w4&HO;j z&uWE$00bZa0SG_<0uX=z1Rwwb2teTf7SOdU&8g6C)2xw8=QMNc&4hdpzyE9IM`C_a zD+B}}009U<00Izz00bZa0SG_<0xKY(XVe@3)ificrT~m^1|a_b{|zy3RveB(P4?hGks{&kHVtq1XEXEQS6m literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2b/data/a/b/0000/0000 b/tests/fsck-data/test2b/data/a/b/0000/0000 new file mode 100644 index 0000000000000000000000000000000000000000..c0605f5fadd20232bdca3ce91ff753274991eff4 GIT binary patch literal 2295 zcmXBWd0Y<$7{~D^6iSY?T7(=Ciq&$q@;D<$h$cy{oH-hiFb{G>h~_>bM-nZOhPdPOP+!Zp0k+g75oo}R(t(C@ zAq!}v#`gd_>irUE%qu?vP2`$MB|}(Kt!V``qe1q7J;%5L%_X-x&_bNexTrofHNl*11|FR3-F`5m{vA~xl*VN z(2mm_fcEma1JFU;Jb{k%t1r-rH3;Y|l|lhG?GXX|M9X6Ucg~6fJj5&o=%W440Zh@^ zKv%x~1n4G~B|vxetpIw^7V|2Gu%7(D7U(4n905-abOU-*oEOlC3;X~tX+8<)s}sY3 zew4Tz=+AF90t2M&e!yEpQh|YV>>@CTKimNZOUHb`N5e~jA(UZcZ0H%v-)aNC($x<5 zS(mg0hEb*m;K$Vl0K>&A0PxqSU|V^bh6x}@mjOKb*fH5*6 z4;ZUE3xRR;>^%^~O^r@~kc)}`u(p3UT23)sS&8v8k{1a|Ym%|M*IjtBN=n>1iAg=7Nz_-HP$ zU)~o0@!FvbI6&c*O$}iQeA)syC>4!?L}h2-5G`TgFkkWplB8-N@VoYz1|(C|0^kVW zhyqfi_IBW?4o(D)(S~#&mGiQI<6?6UNK^lpzzN#<5je?DP0S2o>C(gsIHlw4fzuT4 z3S{uB?!Xyo?E{?Esbc`8Hu74^Nzp`@%996>N;N_n{tAHTWl2y+?HVxz#ZKh z1LV-7I3SlBr2v1*=yO1x?#Tx3QsEQe9y^o(_hoVg@IViln;XI&Qi(0_h@BmQ$1>9m zc%rFZz*G9<2R!4>lYr+kKMcs%Gs}SjGTH?E#ohM5lxBmdvAK@(k literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2b/data/a/b/_format b/tests/fsck-data/test2b/data/a/b/_format new file mode 100644 index 0000000000000000000000000000000000000000..ca49cc290f2b7e665cd978ca27ec8dd76a3a715c GIT binary patch literal 99 zcmZo*sx4&Dh~Q;lU?|EjFODxrEs9Ue%t_ARwW?qR!ykQ{=m{*osRGgWgSIFwkT&f2Eujm|S literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2c/data.sql b/tests/fsck-data/test2c/data.sql new file mode 100644 index 0000000000000000000000000000000000000000..0df1b652680f4fb068a4ea71fe87d86480f4fce0 GIT binary patch literal 24576 zcmeI%&u-H&90zbaOWS3v3mm4XswP?5p_|mL8oTTUg+-{MbgXpJc8WaJT_e(@q%LE} zYVW|y@B};pS0v7$9VSkI^QSEpFo_eQ`&vot*v_Bd$G#+faaebyV4WcBamjYcU7~5^ zA!CG)Y`Uu~PP#fq`#4cmSQwxDAv7b{ zWyw*Wi8!hIyLhKhya8qWDmQdV?$ZwjH}r{a^Sd4i=ZMpsRV>oWLL6qIGDtIRCeXBU z8tG3%`n*|tR&AcJr}jzNFt)aoX*BArwm%u;K9}8OkdGR*!y|h-)=~ecRq1b^ZL?Cv z7w0nYEB#RPOOxw8ei01i>@sAz)IStQIT0 zayoQ2lF|Xz< zK!gJU2tWV=5P$##AOHafKmY;|xYYv7G;1vCi<-IqZbZH(>;KopyuQ^a2p$3ufB*y_ z009U<00Izz00bZ~hXO`M{Q@weKLFACzh-_U<`;E9f&c^{009U<00Izz00bZa0SG|g eKNVQkbb8Oyx~}YsP^?hhu&l1|yx=?tz4i}|R{#kB literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2c/data/a/b/0000/0000 b/tests/fsck-data/test2c/data/a/b/0000/0000 new file mode 100644 index 0000000000000000000000000000000000000000..c0605f5fadd20232bdca3ce91ff753274991eff4 GIT binary patch literal 2295 zcmXBWd0Y<$7{~D^6iSY?T7(=Ciq&$q@;D<$h$cy{oH-hiFb{G>h~_>bM-nZOhPdPOP+!Zp0k+g75oo}R(t(C@ zAq!}v#`gd_>irUE%qu?vP2`$MB|}(Kt!V``qe1q7J;%5L%_X-x&_bNexTrofHNl*11|FR3-F`5m{vA~xl*VN z(2mm_fcEma1JFU;Jb{k%t1r-rH3;Y|l|lhG?GXX|M9X6Ucg~6fJj5&o=%W440Zh@^ zKv%x~1n4G~B|vxetpIw^7V|2Gu%7(D7U(4n905-abOU-*oEOlC3;X~tX+8<)s}sY3 zew4Tz=+AF90t2M&e!yEpQh|YV>>@CTKimNZOUHb`N5e~jA(UZcZ0H%v-)aNC($x<5 zS(mg0hEb*m;K$Vl0K>&A0PxqSU|V^bh6x}@mjOKb*fH5*6 z4;ZUE3xRR;>^%^~O^r@~kc)}`u(p3UT23)sS&8v8k{1a|Ym%|M*IjtBN=n>1iAg=7Nz_-HP$ zU)~o0@!FvbI6&c*O$}iQeA)syC>4!?L}h2-5G`TgFkkWplB8-N@VoYz1|(C|0^kVW zhyqfi_IBW?4o(D)(S~#&mGiQI<6?6UNK^lpzzN#<5je?DP0S2o>C(gsIHlw4fzuT4 z3S{uB?!Xyo?E{?Esbc`8Hu74^Nzp`@%996>N;N_n{tAHTWl2y+?HVxz#ZKh z1LV-7I3SlBr2v1*=yO1x?#Tx3QsEQe9y^o(_hoVg@IViln;XI&Qi(0_h@BmQ$1>9m zc%rFZz*G9<2R!4>lYr+kKMcs%Gs}SjGTH?E#ohM5lxBmdvAK@(k literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2c/data/a/b/_format b/tests/fsck-data/test2c/data/a/b/_format new file mode 100644 index 0000000000000000000000000000000000000000..ca49cc290f2b7e665cd978ca27ec8dd76a3a715c GIT binary patch literal 99 zcmZo*sx4&Dh~Q;lU?|EjFODxrEs9Ue%t_ARwW?qR!ykQ{=m{*osRGgWgSIFwkT&f2Eujm|S literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2d/data.sql b/tests/fsck-data/test2d/data.sql new file mode 100644 index 0000000000000000000000000000000000000000..d059842c1e09a49db45df4c4dd0110b90539bf18 GIT binary patch literal 24576 zcmeI%&u-H&90zbG>z`%NhRYOH)g((hY?Jy&W0&2aun1L@j+IW@PLZd&Yebrq)Me~g z?Hza-o`6T-#)UIzhlvy5#A!oz!lx(IgE;$XVZU zJG9PFCVFa+kDK+QV|zB%QU9n_>Q~MybgSy~ z3laF$LC6PN)9XEU8H~jIG>IRTH288xCU@w10TX-ooqZK!S& ar9fU!8J5-Ko)?H74|!gjo?ir^*ZBc&`~g+~ literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2d/data/a/b/0000/0000 b/tests/fsck-data/test2d/data/a/b/0000/0000 new file mode 100644 index 0000000000000000000000000000000000000000..c0605f5fadd20232bdca3ce91ff753274991eff4 GIT binary patch literal 2295 zcmXBWd0Y<$7{~D^6iSY?T7(=Ciq&$q@;D<$h$cy{oH-hiFb{G>h~_>bM-nZOhPdPOP+!Zp0k+g75oo}R(t(C@ zAq!}v#`gd_>irUE%qu?vP2`$MB|}(Kt!V``qe1q7J;%5L%_X-x&_bNexTrofHNl*11|FR3-F`5m{vA~xl*VN z(2mm_fcEma1JFU;Jb{k%t1r-rH3;Y|l|lhG?GXX|M9X6Ucg~6fJj5&o=%W440Zh@^ zKv%x~1n4G~B|vxetpIw^7V|2Gu%7(D7U(4n905-abOU-*oEOlC3;X~tX+8<)s}sY3 zew4Tz=+AF90t2M&e!yEpQh|YV>>@CTKimNZOUHb`N5e~jA(UZcZ0H%v-)aNC($x<5 zS(mg0hEb*m;K$Vl0K>&A0PxqSU|V^bh6x}@mjOKb*fH5*6 z4;ZUE3xRR;>^%^~O^r@~kc)}`u(p3UT23)sS&8v8k{1a|Ym%|M*IjtBN=n>1iAg=7Nz_-HP$ zU)~o0@!FvbI6&c*O$}iQeA)syC>4!?L}h2-5G`TgFkkWplB8-N@VoYz1|(C|0^kVW zhyqfi_IBW?4o(D)(S~#&mGiQI<6?6UNK^lpzzN#<5je?DP0S2o>C(gsIHlw4fzuT4 z3S{uB?!Xyo?E{?Esbc`8Hu74^Nzp`@%996>N;N_n{tAHTWl2y+?HVxz#ZKh z1LV-7I3SlBr2v1*=yO1x?#Tx3QsEQe9y^o(_hoVg@IViln;XI&Qi(0_h@BmQ$1>9m zc%rFZz*G9<2R!4>lYr+kKMcs%Gs}SjGTH?E#ohM5lxBmdvAK@(k literal 0 HcmV?d00001 diff --git a/tests/fsck-data/test2d/data/a/b/_format b/tests/fsck-data/test2d/data/a/b/_format new file mode 100644 index 0000000000000000000000000000000000000000..ca49cc290f2b7e665cd978ca27ec8dd76a3a715c GIT binary patch literal 99 zcmZo*sx4&Dh~Q;lU?|EjFODxrEs9Ue%t_ARwW?qR!ykQ{=m{*osRGgWgSIFwkT&f2Eujm|S literal 0 HcmV?d00001 diff --git a/tests/test_fsck.py b/tests/test_fsck.py new file mode 100644 index 0000000..ab05b46 --- /dev/null +++ b/tests/test_fsck.py @@ -0,0 +1,118 @@ +import nilmdb +import nilmdb.fsck + +from nilmdb.utils.printf import printf +from nose.tools import assert_raises + +import io +import sys +import shutil +import traceback +from testutil.helpers import * + +class TestFsck(object): + + def run(self, db, fix=True, skip=False, checker=None): + """Run fsck client with the given test database. Save the output and + exit code. + """ + if db is not None: + recursive_unlink("tests/fsck-testdb") + shutil.copytree(f"tests/fsck-data/{db}", "tests/fsck-testdb") + class stdio_wrapper: + def __init__(self, stdin, stdout, stderr): + self.io = (stdin, stdout, stderr) + def __enter__(self): + self.saved = ( sys.stdin, sys.stdout, sys.stderr ) + ( sys.stdin, sys.stdout, sys.stderr ) = self.io + def __exit__(self, type, value, traceback): + ( sys.stdin, sys.stdout, sys.stderr ) = self.saved + # Empty input + infile = io.TextIOWrapper(io.BytesIO(b"")) + # Capture stdout, stderr + errfile = io.TextIOWrapper(io.BytesIO()) + outfile = errfile + with stdio_wrapper(infile, outfile, errfile) as s: + try: + f = nilmdb.fsck.Fsck("tests/fsck-testdb", fix) + if checker: + checker(f) + else: + f.check(skip_data=skip) + sys.exit(0) + except SystemExit as e: + exitcode = e.code + except Exception as e: + traceback.print_exc() + exitcode = 1 + + # Capture raw binary output, and also try to decode a Unicode + # string copy. + self.captured_binary = outfile.buffer.getvalue() + try: + outfile.seek(0) + self.captured = outfile.read() + except UnicodeDecodeError: + self.captured = None + + self.exitcode = exitcode + + def ok(self, *args, **kwargs): + self.run(*args, **kwargs) + if self.exitcode != 0: + self.dump() + eq_(self.exitcode, 0) + + def okmsg(self, db, expect): + self.ok(db) + self.contain(expect) + + def fail(self, *args, exitcode=None, **kwargs): + self.run(*args, **kwargs) + if exitcode is not None and self.exitcode != exitcode: + # Wrong exit code + self.dump() + eq_(self.exitcode, exitcode) + if self.exitcode == 0: + # Success, when we wanted failure + self.dump() + ne_(self.exitcode, 0) + + def failmsg(self, db, expect): + self.fail(db) + self.contain(expect) + + def contain(self, checkstring, contain=True): + if contain: + in_(checkstring, self.captured) + else: + nin_(checkstring, self.captured) + + def dump(self): + printf("\n===dump start===\n%s===dump end===\n", self.captured) + + + def test_fsck(self): + self.okmsg("test1", "\nok") + + def check_paths_twice(f): + f.check() + f.check_paths() + f.bulk.close() + self.ok("test1", checker=check_paths_twice) + + with open("tests/fsck-testdb/data.lock", "w") as lock: + nilmdb.utils.lock.exclusive_lock(lock) + self.failmsg(None, "Database already locked") + + self.failmsg("test1a", "SQL database missing") + self.failmsg("test1b", "Bulk data directory missing") + self.failmsg("test1c", "database version 0 too old") + + self.okmsg("test2", "\nok") + + self.failmsg("test2a", "duplicated ID 1 in stream IDs") + self.failmsg("test2b", "interval ID 2 not in streams") + self.failmsg("test2c", "metadata ID 2 not in streams") + self.failmsg("test2d", "duplicate metadata key") +