diff --git a/ansible/group_vars/all/vault.yml b/ansible/group_vars/all/vault.yml index a2b0fcf..a619094 100644 --- a/ansible/group_vars/all/vault.yml +++ b/ansible/group_vars/all/vault.yml @@ -1,191 +1,194 @@ $ANSIBLE_VAULT;1.1;AES256 -32393162393333376133353565666661663262306566653830666366303432343431613035666166 -3638303965376639353538363166386236363033633165660a616638323233306330663432383233 -37316365353864356130653431323361303137383935356535326434326465336463633166376330 -3731336431306635640a633233353865303466393561396431366133643332653738663164643739 -36323936383031623134353531343930343964393438656238353836306538386633373464306563 -39323735646131373736616638336638343564376433386562326364353135373639316661653233 -66656161366438323966323435343130323265306262663035346563643965613230353563636561 -62653732633263363833343535626563663263356665383864356363393865396133346438663363 -30653332653935376264633165626463623033386633336435353636303238393332376163366532 -32663136383164636262376461303937326561383537383563366663313264316234633366303435 -62313965623731363262303837636239323466356362636236396232636666386365393064376538 -34333136656139653866346563376134316662343536656531363038343061373834313638393936 -64663165323635346133333066393662323163303462326561653735343331373138343565363932 -66396230356565346239346533616261633765316263653738393839343433336631363736643133 -34636533326362663365363933633664386266633539643733316163623933396262393235323962 -31623865343734643435343134373237346466356362316434376133306633653866343437656632 -34656261343861346463323932626135326666393565626537666365636537303131366434336231 -65363737376533616638356565373531643933653061303661373431306433333439346365356337 -61333635616137363137366331643965633261393538333631613830646436303432373062313434 -38373638613364313864663331386635633731663930373063633430303465366532656338356364 -65343631336338346438653239656134633239613037663261336134386266363562646134343630 -38346634613362326630613132623262333232373366343333376130333239323861353936666561 -31363638323461313236623238626366353636356339363633626162363539333139373366616630 -39653231623538663766666339666166333433646432663664336339666233363535346161636461 -37313066643964343664663336346332343463323937656635636239623265316431306330393137 -31336261343731616362303963376331316233303030613035653863383539376131333865333066 -65346639643233346661646563386364363334316661613531643938356439383766373132313030 -36363331383738333332643935613766393965333462383638653239316333383636613363303530 -32666536356332373737396131663531653938396161626266633062646664613963653630623032 -37373862636531643033303335383361333435353964376361376337613662366536326561623233 -38616637613136323538356666613531656635633166383739653239393266316661326634386563 -31326562313062343730643531663463383437626637313439333336616530363031636661366635 -38653637616235613136353138666665313031323735373364663164616636313461616236646239 -36613563316133316466313933303032316330353338636231383739613530666337663363303439 -38333061353966356538376566646138343831623131366466346338666461363066643162616638 -33393231383734386235653233616232346666323962636339313037643636346561363266353464 -38333563306663373664373833636265653135343465333134393732383936353665313165323339 -38323061323035643166356634373736376231663632346263336230303564323766333632353131 -64646164646166336435353064386166313236306165343930656332633365333538383133393037 -61623665396632333462393337653436316634656530303432316238353461643231393238396161 -33313736383463356538393836306132383738393338353036303164363938646532396635313034 -36303862313531353165653034313135393065306662326130353732353461333533383439663866 -30343539353539643036323363303638646338366164303239363766316562633936653330393466 -36393133303162613531643832303239303038323034383133313531633765643234666366643365 -38653831376334656466363564626636303931633434633337343430616230353938663539366434 -64633737623066653632313739373766333463363838366562333835326464653635653635663064 -37376466643566643737313265306237323663363761653538636462613636623666396531333063 -66663037306365383831316363633666666162356461623363613163373632383736353233393363 -39343962623835343335636262306663633266326363313735393766623236343133653032373265 -34636665633939613837666665383735346135373634643266393531386666666137626231396638 -38333166353863333938383163353431636630633037623139356437626464626330346138393832 -34326339373363313763343032343431643165616235376165323264383437666266376261616135 -35373730643837363331626138666565353033316132613631623931353264393837613436343334 -34393137376661343731653934386132326435343931383937366164356336313630663665643263 -64316265643139316433313663363130373766653337636237636638623231386161386238353664 -66663832383137333463396230316431353338353431323133643238363335653734323562306639 -31363162643630616336343362346132313134663236653664653461616566653433323763656333 -64333865626533323834323130343434626134303136643136326366316134366363306433303666 -32663064326235366330303030373163633836306435323630316563376161643331636161303365 -64313433326239386433656139313030323235333538666632663432313439393837666133313566 -62633666333964353031366234636134333737613939663463333632386361316239636339366565 -64393930313665343366623738313561306234316666316532373839653465336339393838326636 -61343638666263643434363862306364663162613637323931623661303932346262363037363963 -34303739646236643830336139656230346436396339373936393130323364633436363039393132 -32643365396134383666353935643530346130613563373731633133396365373364393466396339 -33366663663963376565313630663937333630376661356365356362643639656538336162363831 -62653764306333333137383435626639613035396232303434666163343430313933303839343631 -36373861363333613932653334326238633431663537313863333830323737373463383662323239 -65316161623933663134306430636435653661386333396262613737306537356266616338643637 -37636662373762393538333261623666356532633436383835393763386438393433336339613266 -34613832353363303132326664393064363961623130653439383031363163316132656665333337 -64613736353130376635646432343663653033376539616262383361343866386261333262653333 -33333136363366303564643633306634366361646261396230663231373231323163343139646262 -37333436393332316138643732303862646631643864613434373038393535333036366136363066 -32616331373761613766393933646536363137623238633865336466633639366562643236316163 -65383837303164323036353137393231396234313630623837663537346335343430343064373364 -36333365653936656539303634663066303362383737363130316663656162393733646338393432 -39643733666335376266396465623438656131373737613939656238323837653865316331353739 -33613334313037616533666230376536396230383334653338626663663261646234356237633665 -38383731663939353735646637646637623161353330653134363765376663356164646230353632 -63393634313633663365626634663137396265346265613163636538323031383064613165366435 -31396262336362346130303133373234336263653832633936316664666430393935393064356464 -38323663336434663963393637636234646636656636386263333438626337356230383138613439 -30343838633463663631326462646233656233313735383136343633633832393434366436656531 -34336132356464353232636664666432656261313537373635323430393934656633643961633465 -34353034663032613735343135323361343234663363306432393439636331346337656135316639 -61333235366465343466366139356536616334376362643464323230663037383837303064353833 -30653038383832393237316239633535366665363631363231373232353261663362336361616664 -33656131643263373465353730646561656464363535333361666631663433323638376364633831 -31306666343430366330623338616461626135363862656630336566306432626564616234346433 -64353639353162306464313433323439356130653065316134626435363938656331623831623333 -31656130316565663066363839343466303265396561636665623739356562383366303835636561 -39356433623230366234393332613364393530653731333133666165666538623662623666623061 -38356364333939666139626463633133386336633330336165386237616537353862373666373130 -32333135396634313066386637316330326633326465653463653839363361393833386264346530 -30363962323338333932623130643162386633386532393031626630386364366362306564356537 -38333461643836613036643239393636623832363966636131323131373332336434343062613764 -66313236613439616338373263336535363134343562356231613166623432393536343435346565 -37383966333665633666356462663538343930366163653964626266363938613436373261353339 -31366636626338333432663730633035353935623437353231636331633064303430303765623236 -36386434633437393066333336393534356438343562326665326563353732376566626630636165 -61356139366232333737376136663266666139336664623034323536663261366636396631393162 -64663834366465383562386632343537303263353565373534336430343266356630333665363439 -32663334336565633533623533336365313035643234363930323431393533623933626637393866 -31653432336635363938663039613630623239656566623839653062643034393032316235373363 -37643863373636666639386337666332633432366539306463346138323865306465643834343138 -31363264343038346161636563653962623130373836366666366135353430666266386361346564 -64343630623038626437623532353066336665636332316238323364633162653230393862663933 -65643466313139326566613862363833393534343536656132653730623333643732313438653261 -37633031303633666531663636616336313638306261663532366236386536616465666162623665 -36316231666166613264303938616262396234343464383433653331633466303530623334666566 -32623663653132616334646634326435363832346366366139336661366336353931326232383332 -35666566663763646164393962396535333630393366373737653562373937303965306265323562 -32623730663861323363663138343138613634313839323232313130353138643134616265636437 -62613062616661383262643632303839356535373232383265363565663830363736383730636430 -35666332333961303165653662336261343837366430626530306236656332393465363934366162 -32383439363934396336633430626366653963383134326636613333373536393234353938383631 -64616632393434666135656339656266613064313237346634353831336664666463373864633231 -38663039363632373733646534636262613132633739306531663439313836313761656139353930 -32306232626361616339383931646338313532366161356634373965656139356137616238623463 -37323336323131653335636565393133313162616130343039346536353333343635383234646338 -33363838646265386438353266623732306633643362333866613537313064356631393135653637 -63306432643164333365343461353166646462373131383935653533613433306539633865373363 -30666239373563646132663734616637383965343539643330356531376661633933396233616462 -66383739376236386362623531303531343735653438303031343530363330386434666137616239 -37626333323766626130623039633462616330383934393830623430393466656164373437346166 -64633234633661663736626335323061313431313638363439653736326563393362373736613036 -64636536363865626361313465326365316564353761353834623235343939376231393831333362 -36303532356266306434656538336663383163306631663836313532343335363966663137373033 -32303036393739653530653162653038373663613536366162353863326663636431623033306232 -63353437653936383933303733663235383835623364653637376566353134666239633836613365 -36313362653237373934633831636161323161643762623761643433373466383132613034663538 -31373561336232326463373761623537663833366534353937336565316133623566633138373230 -63643838653330626565363966373735656334313834363034633066393934653331366161396136 -34646536343230306534663661646132306537663132393265663636356638636332363730383961 -30396633333962353835636134343539346332376566646335353533303331363434623635306463 -33636535343938396533623961343465353131623338626233656632333733313732363633646130 -66613230306233383435326531363632313239306566373261373031663136393334626133303761 -66363263306139343837626433656237376631646533653963333834633138353234396631333239 -31323161373435626233643763643965656465633034373061356263346635376633343836333330 -38343139663938363036336161613364323161633163643061336136316663313666336161303733 -39646139663131313537303735653665333265323531353839393432656361633761373433613064 -38653233346433353133306266623534386538393466323334643464306632663336373166633661 -35313831616663353831636663616339643631303163343637656132313236616563373861396135 -65633234396439373431663931343134393739633733393631663766653530373031306363623462 -36383063336666326232373638333430316535363630623630303433393737393431633832316232 -61343363366634663237316666636165353838343261613231373363626633323235306162653165 -64383934636634653134366438383138346233383366386463626663313033313537383031333631 -35633834633639323038666637313431326564346566376333353931336232313739313239373566 -33613862616634663630336532616530643165366534613230623133653931383064363632623538 -62616130653135366533613337343030633663313030646630373633636339386661663762393539 -30396337323932663032346530303034313636326630643966363566643965633362633631333839 -63346461646230663337303537366233613532313032386264303736366262343937393261303264 -64323666306532346530313438323338643830353832373633653334316432353666383134313465 -64313962343464396131636466613130643530383738663564323734323534386435353231306463 -37633734306134386539666165366531656234333863653535323730323537613537363834363536 -30633065376437323364373538653765646433373733356163346531646665616134626533323466 -39343030616264646233633838343233396636626461616561646664326534653762363531396463 -31366163363136316338356135653565316363666537376438666136646635373366633338356363 -66663336643366366436643265633034626239323437396165653263653434313833393032393436 -37643334633639633265393865363731346561366564383430303332343734623130383563376661 -38386135653163343163653930396261376539646238636139616236303130363561326364663962 -33623739363235333633653865623436636437616435376439326662613037313730346266623432 -33363361666138346365313234623631383264356532643433353535386562366237373734623935 -32326666323363353132383230303633383264616138656363636261393734613133386462633436 -30303830336365333664376565313734666132373066396137663431333435373738366464313863 -30393132313937353237636437343935613861313635643637383237643531333638376533316364 -62646631306231346437663234363832393162383738356235663561376333656333646365323433 -66386561373935316662663130626632323930623161383964613830303538383836396135383266 -39613536316132653666396161363164333864343963366634356632383836656662663366613461 -31343964386333653631323733633866333531366336303062353837383366613434333136633830 -65653065646633613864643266323037373231373936646465356663383838373536623461393765 -36386565393038626661623264333036623561663362633161363064383263633931373265366539 -30373137383364326466666532313266343361633436316439623564653065353534373561313265 -39383630383666633138616666633031656663333734636131363338616135363136393366333033 -33396664613130383161333862633766656637383063643230393661323233306364626230393937 -38616438313664383262353735636138316538653930626639653232393531333738313564303731 -32306238336166333934336437303636643135346238386165616539306432336639343130353935 -66383336663738336562366630653330646330363063643632363837666233656236323237383732 -36316536613866623532633233393931333339656365343064393862623662306634336466383764 -65303737373430316238336139663637383133646438633833346339643239623836343138393933 -33636337363130383064366463313439386164316133666238346237313761333437663730373061 -65363733353863396163346534306233633764326462666234373732626232623361626438376336 -65653437353832616537653338356234666238663533373831633065356562373832666238363464 -31616139386533616132313534373534656335666230656264636531306163376164333265333634 -39366539326633613966366464633535363432303238343533653131396335306639623035393434 -65366134316636663238663862366165306132323339393334306435306465373636306337623664 -36666139613361306234366237356634366563666438346537646564653464376330303165316464 -65333030656639313263316637383233373630666536636335313337656266656164326465623030 -3264653937336633613533613866386139613665366337373262 +66646566313430366435336438316665363536356431633739373931326363363137653964333765 +6438363638393236306464663136663061386361363432390a623836393662636238626337373766 +39313636386137363334313465373637613363313634363230643032386132366533323934393933 +3033616238356165630a613765643762336161663863393163616636636163376537396638666139 +62376334383839373131646334383039396661396262633537333863613133346363303133353765 +61323462646434313539633833616162376637613436626365336265333863356534626339313661 +35376634386233653964353437373838343066323035366263663738643032646631633236663066 +34383535333961313439386462326264616535366434343032633639646631373930353038356330 +37626465653362366666316363333663343734663939393637373234303935363136366636306534 +65356362313662356332626530363563333337613661626531326138633936316137626334616162 +30336664323436653431393061343935386461383130356437646361333431313938393236363633 +39303063653936353736363332386366623565313861613838303735356530663138323935323066 +66613134653235353364366263343138666639396430306439396633393433636236363363393164 +66326330313530393064656331386233363466386336313430333139643830626535653538353331 +62643631366536653137393733666238386230383634356532663563343765306334383331333161 +35313336643266653231333034626139636133653236323235346238323335663934626133616664 +65326333323334663265303933353035353739323062323962396661666161333236666133343235 +65616562346131613136363932353632343935353139386234656138303261636230333264363230 +33333933623965343862393065313238306635636361373237626233353634373737336532626431 +30636163363835366330326163616539303031393561313538393338323166613739663039396566 +61613265653633386338663837396135356563353439336534633064363330303863333166393966 +35366337353636366336346436633734363833363133366561333835306433333138666134306462 +38626333663534386237303739646664663363303866656433313733393832316533383136316233 +66363234656664346465636562643832323266346631653262373166343036363532646533313636 +63303538656662383430623365383133313665393163333832626361386663346132313163313061 +39613732366435343930346639643161656165656538653531653036393764663962363832663861 +33333031353234646239386534623536376131306261653165393634383762383265313136326636 +33333430323030636164346433316337663137643133626639303536323735623637333031356633 +36613232313330636636633266636461616463376463366234303630613966363738633133636263 +64646532333764343861633935633737323639333237643831353339636235316533613266653865 +36333535333365333436653638663364386437613136313239376538323439613965343339656461 +39616239346435343665333831363134643762646532393366366533623461353964383263356630 +31373239666266613133323333323336363739346236656261323365363664303630663934653863 +34636534376633333331356162393339653762303837363765313633333535386164386439386266 +39303336646465366336316263343461613336303930653963613866663738333433326565643736 +61303631656132653064303439646637373734313364303135643963353064363061663835663632 +31623264383238323734373732393733343336323630366439303466316238336566623666303433 +34396633653762303730323334356166303134376535316131333266303438313562306135343233 +30303136333730626561656639623866666663646131383666613633663966636462663031626639 +65383732313561376261333534643937383165616137656232623664386232353337343563333262 +34633364323839353136616636366462666637313438396661303663316263353636363163633236 +64326265623262323735396637663538626139366165643437333233643830313962663735373063 +37363664363432326637386335386432616135666162363431356561333635616134626136316365 +38323539663966643262656635356334386236393538333034303565393439353931663466373966 +31376466393363396535396263336338306239613961356130613335316132313335636634353461 +35343830633036653337356131386538396139323265653230396664653533636239613239356639 +66316332323538656166393037636563626338386235633230623962373731386230383733366663 +64383064623362626639383030396339623138373935373539643335366531393635316137323266 +61383065353766626463636338383563366532323439616333623730386630656163343536646562 +63343538616139363933316266363531656661633532353765366534383735366664616565623030 +33373561386537613737653764326439366632626537346464363132303133393963313966373163 +32623239326631333338303736613465616435346535336332633163313439656232626239653836 +33343638643562626338353137346365386431663334336231656435373238323535323733623461 +37356533613738663764656466396632366430316433623334666665363463386131303832326564 +62613163366661346435343031373433343236643839396536383966363863393836623265393135 +62303838393539366166383137343737383432333235633230363036656466373837396462393564 +61353535623738616239663535333666323839666133346336646533633735623639376332656464 +35633839633665313033353834383530336164346233643033373336656265363266363534316465 +36346536383535333138313761356561323661643163393831343962633036373265643938336533 +63623135386630356262386563353634623439383631633866366661336336613038623439326264 +39353933643233323863343839656634373732363661373966313839363130646533323737646433 +33626634643964336236646233376235376230616435323930323935373730633663393435646331 +64613839366663663933626537323339613363643839313632313034633135643166323235356632 +66643437353130333831306530653834386137386632626363646563343631333235653065353730 +38323363373665623961306530316562353165656138393134653933303239663636643464306136 +66303234353537633062663735653232366639373030336330336635323065643935323936396333 +64316661323263386664323031333030326534306566333934306361353233373536356435326338 +33303561326233656233366431653237333763326434313162663165633637636262343130636133 +61366635663730653539386366353237633438323966633961303032366136336630623436393035 +62633463313338353432323939343239623630383565636162616139613664613863396165643736 +35376462333938656432373133383838623035653131646366373239306262323539313439343166 +66323830616362363263636663333566386239353765343830343465353265333262313331343434 +30313339303064313263633864623261616130343530336134333661333931663564323461353034 +64663737393430383233633330306565363230383761346362663530646238363631383062646230 +62633432623563353239353737623938663437303061623866363431343436313837343539343436 +30393737346435363563346235633438306339366639613161383763613730353437636537366565 +66396534353230613137326139333435393034346266643364646138666130656532323166623664 +36656631336636393131316361376339383333333837663263343065623937663062346666323464 +31636662386362633337306338633138623434323563643663663835656430333762366631343033 +63343861613365336434653035333132636363656139393065356330376161383039633533393366 +38326630396535383830303437666333623735393131336630663936373465326466616133343735 +34393535326335646265383333646233386434636530613464643266326134396462313762363066 +64303637386261653866623334666532323130393838313034633038356361363239386435656239 +63386163653362386565646561353238653962343033386632336364313630353063356336626338 +63356137616530643434313862643235333063363533666635346661363861383366326533646665 +66316531633163313365653337663633393661313466643331346230613539653663336637353162 +63323730386433626364333632383761323362363664306264336334613766373532386533336662 +65396336313332633539633966313733386335613236666234313664303631326361376232383036 +33663563613864643532363038303832363434386265366637376561373434643239396231643634 +61633935363234383935393333666566383766646461623061613361636531386363363238393465 +66643636633465343033306437626632373734323762666164356231333137376535313831373236 +66373435323339316533383432393766633566306635336166336336643532313430393539336332 +37613666333763396439396466313535383066636237343964306661636337383931636631633463 +64383130613964376462663837383762353634623064313431623266383839333434653537626663 +34363736643937326134616136306236656261313136366635303036646435353863306662353034 +39363266356636623135356232383630373162363330396262323734643564366232333632373739 +36656366633335626131643930366534326164646265643339623539323863393632353838633861 +36363763313039633530623736313239353036666162346432643732343438366265306164383235 +66376536653761383665316665396435646634363436376632646563633163623766366635623731 +66623831393230326566313932653536343838626566623038653534623434313239343763313433 +30376630656638623036336166346133633738303638366238666639653030323832323365353766 +33623261343664323631393166313539623734393438346562613735623636396137333733303235 +34636630643932383536633836343765373234623639643437306532353764386435663665643738 +65313065393534383935303639656534633363303235393761343330386630633734313561376433 +63623464653034336661386435386431306634613239393039386439316538343261306234393534 +30323164363734303037663134663734363636653935336539396330383931353461353966356533 +63316462376262626261623031353063353030363230653966393832626662623963306266393635 +34643565386137663435373639393537626632373535333865366165353839343732353439616265 +37373563326438656134363931623565313565666334323533653264356633353931323661663832 +35396637353163383466653765376132353561343031623039383161623034396134626562666632 +64643737306634613966663362373038396231616632636539663966313364383137326164396533 +38643266316464366161343761633332633036383537653565653139336562666461666465376230 +61656361623264366566656461373863656161393333353933326665333730356333343531653766 +36303964353134396533306630343832303961336239643462353439363962353738663731663033 +38343833323365366662343665633130306537383861613762613763646631626330613937393432 +36393338626661306130353461653565313766623762613564356466393663383339356335306436 +35373636396239653361633764366133363334643232643734383335346635386139383130363938 +35643665316139613639666436633531396537353439633839623063383134376230376235366534 +64393634376633343063646433323439303438303432323335393265366633343332616461386665 +62626132346364373931303936623963663037333665333639333966646431663635303365343630 +36316464346266313166356262346331656432363865643135613964356662366632323736613437 +37336561376334303534653237303964656665643561376237653531653764363463363464313737 +66313335383838333233663162336337373764363131643836363365353039323836346433313139 +33633636306131623335623639666130393535646536346438303435366436623132613562643066 +65623336643463333932383865353232333339343231613964373566653063643662376338316437 +65343466396530386230396236656463396566336230343838366264656161323063336264643364 +31643066366632653962323636346565313833393966343761623066336464333733646238386332 +37393534356333396661353130363462633366343535616338653330393262663066326662366530 +30383536653838333066623063306233613735363665623937373362623337323339373663303933 +64306538636533643337373766346266326238653937613366616161336333636662386238346237 +35623237333131623363376230663132373939366235376631626263393764366661346438383537 +33306233663630323863343839636166303461666135313639356637303435643062303364646132 +64656636396264303962663363376166323834393863393264323464336662393930636533363834 +61363937383766623930623537353037623031363736633332393766323238383464663531653438 +34653933623166326530343632386165666266303438623739356635346163346239643065336537 +33316631613431623030323433306133376237316535306666343433646339313965663565653130 +61303065303530616639383066623534353566366632313465316136613437383765643039313137 +64393333353834353033353532373834343831326535393837323037373136303462373530646430 +62633964626164626139373562616430613662646265396139356561666462376435643031643964 +62316339303561666435663438393838356437646336623664613863303134343261373830303730 +36303732393262363530366563343165386436313833633533623637646464616663353035363235 +31353764353135343033363562313465383563376164323766323735646336353834613264316464 +65376134346437333439376636313531666638313333313963636363353965363239323830663434 +30373234373235326635633063366232333235653935616365656135396465366564313530363161 +33363864303063363363363535323333616661323338323735323135613562366163393366393963 +31363331373661636134333063363731343831333061326431633061663064356466306438646565 +62646431613861393237313138353330323734636666333934666437343665653965316436636464 +64333432313561656339366533363765343365613139626331396636376535326165623735346633 +61666364373465613438643032316362303335333431306166393766663032666665666564313837 +38323232373439376563633939623835656464313361333666623336666461383330663764666166 +34343638646637646338313737336566623937616666626637386639346530616439316635396661 +61656439346231663436386665653031643334393734346163353963633333646239623536333730 +39363731316361633139326637303839633633633738633737326465343664336261646337343831 +33623261363136613935316635663434633962613635616333383238613965646637363735646264 +39626635323532396339653861383137336136346539303931323262663635366432376365623732 +35356639376161323937306136376236643431643863393630633662346531343230643636666633 +33636139323663353835313663656162656264313739663066616663346639326434353762666330 +62303235363964323964663664353265316461373637346230646364393065653763646636303664 +62623735633234303063306531393662326363333234333763613335666233373131303334366665 +30633037323934633232316366663563326562353834643132623031613138323835373933376536 +37323638396561643363663235333632323234303734643637663261393736613738346433333539 +33623865363862653731366466333530343561653661653638353738346438633236343730326239 +35363837393736343632663765626635613865656165396336646538376434653537633037616566 +37393766636661383538386164363630626438616365666532333039663464303835373134333432 +30386439333732666266666331633566326463316438333639336665613933343963363562636335 +31356465303535663233656466616462646161666538346334323335396561656634396631356262 +62323561316261653134363436636333353665613165323935353439666164363163303530353964 +66633536306432323335306530323733623536383366653566653339383161383434613330343934 +63666131333963356336333763386338343437613035313036393766636535386234613665636632 +34383032633931663633386636343233646131393066393766366635343935356538306231336232 +32353836353335363731353234303337343737363064636463613132393563316136626539326264 +30343230343166393466353664613438383462623734363038363962306437663938306235316235 +64373433626366636133373964633131636534333839623033383163613561363430343534313966 +63366561613939623334653434616238616136353665386665346366666236326363376135373363 +36343965343131356662373834366535313963333366633536343930633561626230373736303030 +65373063386232316332343963343631333162376334636665633531313936303232323933313263 +31373639613365396533303431376532643434626538653538356630626532343530663163366232 +39376566353361333639343237653937303161646164613831643235613139373961663566333630 +62663430313832663432306634356464623838303835373962373533313565663735326535326461 +66653939623836666461626631656534616162323433356431303366306635323938373138613031 +66366461353133386436633765386262373736633539643637373237326235633764353132636561 +63343438666361303364333732303738323463353832613438636663363332333132373361633439 +64373334386661626364393664346262653739613134346365636132613538666339393036333163 +64373363643031393062316135336561303665623465336464356165313466633462666331303030 +62626535396634623366313662643066313462636330343237316433646434313737626562346237 +65393161333336386430353938336536336461326132393933343335656131393664356463376631 +38373862363862343839666161663136336534633862303636653931363136303337353664393737 +35316138353361306133636463366463333137616562366135653230343039643365646665636364 +65366564643738356539663331613439656536376263306362366165306230373336386166613865 +33626631623364663062396466626531666238616436326163643533386366633164633262356431 +32313762336561396531386535383434396135633066316334626632633431356363373034663135 +33323166373562326237636265353634623236666364643837353232306263643132343761373230 +63626331393461653662383666646332303632386634313034313664346432656265 diff --git a/ansible/host_vars/nodito/main.yml b/ansible/host_vars/nodito/main.yml index c0002f3..a6f6a58 100644 --- a/ansible/host_vars/nodito/main.yml +++ b/ansible/host_vars/nodito/main.yml @@ -14,7 +14,12 @@ systemd_service_name: nodito-cpu-temp-monitor # ZFS Pool Configuration zfs_pool_name: "proxmox-tank-1" -zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror +# Corrected 2026-09-13: this said WX11TN0Z, a disk that is no longer in the +# machine. The live mirror is WX120LHQ + WX11TN2P - a leg was evidently +# replaced and the repo never caught up. Pool creation is guarded by +# `when: zfs_pool_exists.rc != 0` so it was inert, but it would have been +# wrong on any disaster-recovery run. +zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX120LHQ" # First disk for RAID 1 mirror zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror zfs_pool_mountpoint: "/var/lib/vz" diff --git a/ansible/host_vars/nodito/vault.yml b/ansible/host_vars/nodito/vault.yml new file mode 100644 index 0000000..e64085e --- /dev/null +++ b/ansible/host_vars/nodito/vault.yml @@ -0,0 +1,11 @@ +$ANSIBLE_VAULT;1.1;AES256 +30333035323663393939343061323234336164396465623665346165393534646366333332376463 +3364373463333664363334373964323838336531353364310a636636373539623464336630666164 +61376532616339376562373238383436306664313564663266303534346461666466383965323538 +3163313239626663310a613033336332653165333537313366636361663036383031376561613761 +31313563373062333033323037653939663762343161656264633436343361663737626366663732 +39366666626338323436383134646263643538333564313566346336323563663534653161396136 +39333565393538366238643563323630346166643461643063393631643665363566623631373762 +36643866646637306231653837363838656163613766636265383139333838396535626335343163 +32613935353330636263616333666230323436663935326133636362343836323535623237646235 +6266316363366335323162663039366137633865396237373632 diff --git a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml index 8cdde6a..97e0cab 100644 --- a/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml +++ b/ansible/infra/nodito/32_zfs_pool_setup_playbook.yml @@ -3,7 +3,6 @@ become: true vars_files: - ../infra_vars.yml - - nodito_vars.yml tasks: - name: Verify Proxmox VE is running @@ -139,17 +138,18 @@ Config file exists: {{ storage_cfg_file.stat.exists }} Storage check result: {{ storage_exists_check.rc }} Pool exists: {{ zfs_pool_exists.rc == 0 }} - Will remove storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc == 0 }} Will add storage: {{ zfs_pool_exists.rc == 0 and storage_exists_check.rc != 0 }} - - name: Remove existing storage if it exists - command: pvesm remove {{ zfs_pool_name }} - register: pvesm_remove_result - failed_when: false - when: - - zfs_pool_exists.rc == 0 - - storage_exists_check.rc == 0 - + # Registration is add-only on purpose. There used to be a "Remove existing + # storage if it exists" task here that ran `pvesm remove` whenever the + # storage WAS present, paired with an add that only ran when it was ABSENT. + # The two conditions are mutually exclusive, so a real run against a + # correctly-configured hypervisor removed the storage entry backing every VM + # and never put it back. It would also have dropped `mountpoint /var/lib/vz`, + # which the live entry has and which `pvesm add` below does not set. + # + # If the storage entry ever needs its options changed, edit + # /etc/pve/storage.cfg or use `pvesm set` - do not re-register it from here. - name: Add ZFS pool storage to Proxmox using pvesm command: > pvesm add zfspool {{ zfs_pool_name }} @@ -171,27 +171,24 @@ msg: "ZFS pool {{ zfs_pool_name }} is not in a healthy state" when: "'ONLINE' not in final_zfs_status.stdout" -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# ───────────────────────────────────────────────────────────────────────────── +# ZFS health monitoring and monthly scrub. # -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. +# The check script decides healthy/unhealthy and says so in its exit code, which +# systemd keeps: systemctl is-failed zfs-health-monitor.service # -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. -# -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ +# Reporting anywhere else is optional. Set `healthcheck_push_url` and the script +# will also GET it with ?status=up|down; leave it empty and the exit code is +# still the whole answer. Today that URL points at Uptime Kuma, which is still +# running on watchtower but is no longer deployed by Ansible - the credentials +# were retired, the service was not. If it is ever replaced, `healthcheck_push_url` +# is the only thing that needs to change here. +# ───────────────────────────────────────────────────────────────────────────── - name: Setup ZFS Pool Health Monitoring and Monthly Scrubs hosts: hypervisor become: true vars_files: - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml vars: zfs_check_interval_seconds: 86400 # 24 hours @@ -202,143 +199,10 @@ zfs_log_file: "{{ zfs_monitoring_script_dir }}/zfs_health_monitor.log" zfs_systemd_health_service_name: zfs-health-monitor zfs_systemd_scrub_service_name: zfs-monthly-scrub - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" + # Optional. Empty is fine and is not an error - see the banner above. + healthcheck_push_url: "{{ healthcheck_push_urls.zfs_health | default('') }}" tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "zfs-health-{{ host_name.stdout }}" - monitor_friendly_name: "ZFS Pool Health: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma ZFS health monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_zfs_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - # Get all monitors - monitors = api.get_monitors() - - # Get all notifications and find ntfy notification - notifications = api.get_notifications() - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - # Find or create group - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - group_result = api.add_monitor(type='group', name=group_name) - # Refresh to get the full group object with id - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - # Find or create/update push monitor - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - monitor = api.edit_monitor(existing_monitor['id'], **monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - monitor_result = api.add_monitor(**monitor_data) - # Refresh to get the full monitor object with pushToken - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - # Output result as JSON - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma ZFS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_zfs_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Daily health check for pool {{ zfs_pool_name }}" - "{{ zfs_check_timeout_seconds }}" - "{{ zfs_check_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL and monitor ID as facts - set_fact: - uptime_kuma_zfs_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - uptime_kuma_monitor_id: "{{ monitor_info_parsed.monitor_id }}" - - name: Install required packages for ZFS monitoring package: name: @@ -355,264 +219,41 @@ mode: '0755' - name: Create ZFS health monitoring script - copy: + template: dest: "{{ zfs_monitoring_script_path }}" - content: | - #!/bin/bash - - # ZFS Pool Health Monitoring Script - # Checks ZFS pool health using JSON output and sends heartbeat to Uptime Kuma if healthy - # If any issues detected, does NOT send heartbeat (triggers timeout alert) - - LOG_FILE="{{ zfs_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_zfs_push_url }}" - POOL_NAME="{{ zfs_pool_name }}" - HOSTNAME=$(hostname) - - # Function to log messages - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - # Function to check pool health using JSON output - check_pool_health() { - local pool="$1" - local issues_found=0 - - # Get pool status as JSON - local pool_json - pool_json=$(zpool status -j "$pool" 2>&1) - - if [ $? -ne 0 ]; then - log_message "ERROR: Failed to get pool status for $pool" - log_message " -> $pool_json" - return 1 - fi - - # Check 1: Pool state must be ONLINE - local pool_state - pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') - - if [ "$pool_state" != "ONLINE" ]; then - log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" - issues_found=1 - else - log_message "OK: Pool state is ONLINE" - fi - - # Check 2: Check all vdevs and devices for non-ONLINE states - local bad_states - bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.state? and .state != "ONLINE") | - "\(.name // "unknown"): \(.state)" - ' 2>/dev/null) - - if [ -n "$bad_states" ]; then - log_message "ISSUE: Found devices not in ONLINE state:" - echo "$bad_states" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: All devices are ONLINE" - fi - - # Check 3: Check for resilvering in progress - local scan_function scan_state - scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - - if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then - local resilver_progress - resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') - log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" - issues_found=1 - fi - - # Check 4: Check for read/write/checksum errors on all devices - # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison - local devices_with_errors - devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' - .pools[$pool].vdevs[] | - .. | objects | - select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | - "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" - ' 2>/dev/null) - - if [ -n "$devices_with_errors" ]; then - log_message "ISSUE: Found devices with I/O errors:" - echo "$devices_with_errors" | while read -r line; do - log_message " -> $line" - done - issues_found=1 - else - log_message "OK: No read/write/checksum errors detected" - fi - - # Check 5: Check for scan errors (from last scrub/resilver) - local scan_errors - scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') - - if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then - log_message "ISSUE: Last scan reported $scan_errors errors" - issues_found=1 - else - log_message "OK: No scan errors" - fi - - return $issues_found - } - - # Function to get last scrub info for status message - get_scrub_info() { - local pool="$1" - local pool_json - pool_json=$(zpool status -j "$pool" 2>/dev/null) - - local scan_func scan_state scan_start - scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') - scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') - scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') - - if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then - echo "scrub in progress (started $scan_start)" - elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then - echo "last scrub: $scan_start" - else - echo "no scrub history" - fi - } - - # Function to send heartbeat to Uptime Kuma - send_heartbeat() { - local message="$1" - - log_message "Sending heartbeat to Uptime Kuma: $message" - - # URL encode the message - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent successfully (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - # Main health check logic - main() { - log_message "==========================================" - log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" - - # Run all health checks - if check_pool_health "$POOL_NAME"; then - # All checks passed - send heartbeat - local scrub_info - scrub_info=$(get_scrub_info "$POOL_NAME") - - local message="Pool $POOL_NAME healthy ($scrub_info)" - send_heartbeat "$message" - - log_message "Health check completed: ALL OK" - exit 0 - else - # Issues found - do NOT send heartbeat (will trigger timeout alert) - log_message "Health check completed: ISSUES DETECTED - NOT sending heartbeat" - log_message "Uptime Kuma will alert after timeout due to missing heartbeat" - exit 1 - fi - } - - # Run main function - main + src: templates/zfs_health_monitor.sh.j2 owner: root group: root mode: '0755' - name: Create systemd service for ZFS health monitoring - copy: + template: dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.service" - content: | - [Unit] - Description=ZFS Pool Health Monitor - After=zfs.target network.target - - [Service] - Type=oneshot - ExecStart={{ zfs_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target + src: templates/zfs-health-monitor.service.j2 owner: root group: root mode: '0644' - name: Create systemd timer for daily ZFS health monitoring - copy: + template: dest: "/etc/systemd/system/{{ zfs_systemd_health_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Pool Health Monitor daily - Requires={{ zfs_systemd_health_service_name }}.service - - [Timer] - OnBootSec=5min - OnUnitActiveSec={{ zfs_check_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target + src: templates/zfs-health-monitor.timer.j2 owner: root group: root mode: '0644' - name: Create systemd service for ZFS monthly scrub - copy: + template: dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.service" - content: | - [Unit] - Description=ZFS Monthly Scrub for {{ zfs_pool_name }} - After=zfs.target - - [Service] - Type=oneshot - ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target + src: templates/zfs-monthly-scrub.service.j2 owner: root group: root mode: '0644' - name: Create systemd timer for monthly ZFS scrub - copy: + template: dest: "/etc/systemd/system/{{ zfs_systemd_scrub_service_name }}.timer" - content: | - [Unit] - Description=Run ZFS Scrub on last day of every month at 4:00 AM - Requires={{ zfs_systemd_scrub_service_name }}.service - - [Timer] - OnCalendar=*-*~01 04:00:00 - Persistent=true - - [Install] - WantedBy=timers.target + src: templates/zfs-monthly-scrub.timer.j2 owner: root group: root mode: '0644' @@ -649,9 +290,8 @@ msg: | ✓ ZFS Pool Health Monitoring deployed successfully! - Monitor Name: {{ monitor_friendly_name }} - Monitor Group: {{ uptime_kuma_monitor_group }} Pool Name: {{ zfs_pool_name }} + Push URL: {{ healthcheck_push_url | default('', true) | ternary('set', 'not set - exit code only') }} Health Check: - Frequency: Every {{ zfs_check_interval_seconds }} seconds (24 hours) @@ -672,10 +312,3 @@ - Resilver status (alerts if resilvering) - Read/Write/Checksum errors - Scrub errors - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_zfs_monitor.py - state: absent - delegate_to: localhost - become: no diff --git a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml index 3b687ee..e2b1f5d 100644 --- a/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml +++ b/ansible/infra/nodito/33_proxmox_debian_cloud_template.yml @@ -3,7 +3,6 @@ become: true vars_files: - ../../infra_vars.yml - - nodito_vars.yml vars: # Defaults (override via vars_files or --extra-vars as needed) diff --git a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml index f39fb56..da1136f 100644 --- a/ansible/infra/nodito/34_nut_ups_setup_playbook.yml +++ b/ansible/infra/nodito/34_nut_ups_setup_playbook.yml @@ -3,10 +3,33 @@ become: true vars_files: - ../../infra_vars.yml - - nodito_vars.yml - - nodito_secrets.yml tasks: + # ------------------------------------------------------------------ + # Safety catch + # + # /etc/nut/upsd.users and /etc/nut/upsmon.conf on nodito were written by + # hand in January 2026 and carry a working password. host_vars/nodito/vault.yml + # (formerly infra/nodito/nodito_secrets.yml) still holds the literal string + # CHANGE_ME_TO_SECURE_PASSWORD, so running this play would overwrite that + # working pair with a placeholder and restart NUT - leaving the hypervisor's + # UPS unmonitored and unable to trigger a clean shutdown on mains loss. + # + # Until the real password is put in the vault, stop here. + # ansible-vault edit host_vars/nodito/vault.yml + # ------------------------------------------------------------------ + - name: Refuse to run with a placeholder UPS password + assert: + that: + - ups_password is defined + - ups_password | length > 0 + - ups_password != "CHANGE_ME_TO_SECURE_PASSWORD" + fail_msg: >- + ups_password is unset or still the placeholder. Applying this play would + overwrite the working /etc/nut/upsd.users and /etc/nut/upsmon.conf on + nodito and restart NUT. Put the real password in the vault first: + ansible-vault edit host_vars/nodito/vault.yml + # ------------------------------------------------------------------ # Installation # ------------------------------------------------------------------ @@ -75,90 +98,45 @@ # Configuration files # ------------------------------------------------------------------ - name: Configure NUT mode (standalone) - copy: + template: dest: /etc/nut/nut.conf - content: | - # Managed by Ansible - MODE=standalone + src: templates/nut.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure UPS device - copy: + template: dest: /etc/nut/ups.conf - content: | - # Managed by Ansible - [{{ ups_name }}] - driver = {{ ups_driver }} - port = {{ ups_port }} - desc = "{{ ups_desc }}" - offdelay = {{ ups_offdelay }} - ondelay = {{ ups_ondelay }} + src: templates/ups.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd to listen on localhost - copy: + template: dest: /etc/nut/upsd.conf - content: | - # Managed by Ansible - LISTEN 127.0.0.1 3493 + src: templates/upsd.conf.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsd users - copy: + template: dest: /etc/nut/upsd.users - content: | - # Managed by Ansible - [{{ ups_user }}] - password = {{ ups_password }} - upsmon master + src: templates/upsd.users.j2 owner: root group: nut mode: "0640" notify: Restart NUT services - name: Configure upsmon - copy: + template: dest: /etc/nut/upsmon.conf - content: | - # Managed by Ansible - MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master - - MINSUPPLIES 1 - SHUTDOWNCMD "/sbin/shutdown -h +0" - POLLFREQ 5 - POLLFREQALERT 5 - HOSTSYNC 15 - DEADTIME 15 - POWERDOWNFLAG /etc/killpower - - # Notifications - NOTIFYMSG ONLINE "UPS %s on line power" - NOTIFYMSG ONBATT "UPS %s on battery" - NOTIFYMSG LOWBATT "UPS %s battery is low" - NOTIFYMSG FSD "UPS %s: forced shutdown in progress" - NOTIFYMSG COMMOK "Communications with UPS %s established" - NOTIFYMSG COMMBAD "Communications with UPS %s lost" - NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" - NOTIFYMSG REPLBATT "UPS %s battery needs replacing" - - # Log all events to syslog - NOTIFYFLAG ONLINE SYSLOG - NOTIFYFLAG ONBATT SYSLOG - NOTIFYFLAG LOWBATT SYSLOG - NOTIFYFLAG FSD SYSLOG - NOTIFYFLAG COMMOK SYSLOG - NOTIFYFLAG COMMBAD SYSLOG - NOTIFYFLAG SHUTDOWN SYSLOG - NOTIFYFLAG REPLBATT SYSLOG + src: templates/upsmon.conf.j2 owner: root group: nut mode: "0640" @@ -250,28 +228,33 @@ - nut-monitor -# ═════════════════════════════════════════════════════════════════════════════ -# DEPRECATED — Uptime Kuma was decommissioned on 2026-09-11. +# ───────────────────────────────────────────────────────────────────────────── +# UPS heartbeat monitoring. # -# This play WILL FAIL if run as-is, and that is deliberate: uptime_kuma_username -# and uptime_kuma_password were removed from the vault, so the "Validate Uptime -# Kuma configuration" assert stops it before anything is installed or changed. +# The script decides on-mains/on-battery and says so in its exit code, which +# systemd keeps: systemctl is-failed ups-heartbeat.service # -# It is kept because the CHECK LOGIC is the durable part — what gets measured, -# the thresholds, and the systemd timer plumbing. When something replaces Uptime -# Kuma, only the push transport needs rewriting; the rest still applies. +# Reporting anywhere else is optional. Set `healthcheck_push_url` and the script +# will also GET it with ?status=up|down; leave it empty and the exit code is +# still the whole answer. Today that URL points at Uptime Kuma, which is still +# running on watchtower but is no longer deployed by Ansible - the credentials +# were retired, the service was not. If it is ever replaced, `healthcheck_push_url` +# is the only thing that needs to change here. # -# What was being monitored: archive/uptime_kuma/MONITORS.md -# ═════════════════════════════════════════════════════════════════════════════ -- name: Setup UPS Heartbeat Monitoring with Uptime Kuma +# NOTE: this play has never actually been applied to nodito. /opt/ups-monitoring +# does not exist and there is no ups-heartbeat timer. What is on the box is a +# hand-written /usr/local/bin/ups-heartbeat.sh - mode 0644, not executable, and +# referenced by no unit and no cron entry, so nothing has ever run it. Its push +# token (uLmCPkLLO4) belongs to a monitor that DOES still exist and answer, so +# that monitor has had no heartbeat since January 2026. It is now +# healthcheck_push_urls.ups in the vault, and running this play is what will +# finally start feeding it. +# ───────────────────────────────────────────────────────────────────────────── +- name: Setup UPS Heartbeat Monitoring hosts: hypervisor become: true vars_files: - ../../infra_vars.yml - - ../../services_config.yml - - ../../infra_secrets.yml - - nodito_vars.yml - - nodito_secrets.yml vars: ups_heartbeat_interval_seconds: 60 @@ -281,134 +264,10 @@ ups_monitoring_script_path: "{{ ups_monitoring_script_dir }}/ups_heartbeat.sh" ups_log_file: "{{ ups_monitoring_script_dir }}/ups_heartbeat.log" ups_systemd_service_name: ups-heartbeat - uptime_kuma_api_url: "https://{{ subdomains.uptime_kuma }}.{{ root_domain }}" - ntfy_topic: "{{ service_settings.ntfy.topic }}" + # Optional. Empty is fine and is not an error - see the banner above. + healthcheck_push_url: "{{ healthcheck_push_urls.ups | default('') }}" tasks: - - name: Validate Uptime Kuma configuration - assert: - that: - - uptime_kuma_api_url is defined - - uptime_kuma_api_url != "" - - uptime_kuma_username is defined - - uptime_kuma_username != "" - - uptime_kuma_password is defined - - uptime_kuma_password != "" - fail_msg: "uptime_kuma_api_url, uptime_kuma_username and uptime_kuma_password must be set" - - - name: Get hostname for monitor identification - command: hostname - register: host_name - changed_when: false - - - name: Set monitor name and group based on hostname - set_fact: - monitor_name: "ups-{{ host_name.stdout }}" - monitor_friendly_name: "UPS Status: {{ host_name.stdout }}" - uptime_kuma_monitor_group: "{{ host_name.stdout }} - infra" - - - name: Create Uptime Kuma UPS monitor setup script - copy: - dest: /tmp/setup_uptime_kuma_ups_monitor.py - content: | - #!/usr/bin/env python3 - import sys - import json - from uptime_kuma_api import UptimeKumaApi - - def main(): - api_url = sys.argv[1] - username = sys.argv[2] - password = sys.argv[3] - group_name = sys.argv[4] - monitor_name = sys.argv[5] - monitor_description = sys.argv[6] - interval = int(sys.argv[7]) - retries = int(sys.argv[8]) - ntfy_topic = sys.argv[9] if len(sys.argv) > 9 else "alerts" - - api = UptimeKumaApi(api_url, timeout=120, wait_events=2.0) - api.login(username, password) - - monitors = api.get_monitors() - notifications = api.get_notifications() - - ntfy_notification = next((n for n in notifications if n.get('name') == f'ntfy ({ntfy_topic})'), None) - notification_id_list = {} - if ntfy_notification: - notification_id_list[ntfy_notification['id']] = True - - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - if not group: - api.add_monitor(type='group', name=group_name) - monitors = api.get_monitors() - group = next((m for m in monitors if m.get('name') == group_name and m.get('type') == 'group'), None) - - existing_monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - monitor_data = { - 'type': 'push', - 'name': monitor_name, - 'parent': group['id'], - 'interval': interval, - 'upsideDown': False, # Normal heartbeat mode: receiving pings = healthy - 'maxretries': retries, - 'description': monitor_description, - 'notificationIDList': notification_id_list - } - - if existing_monitor: - api.edit_monitor(existing_monitor['id'], **monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - else: - api.add_monitor(**monitor_data) - monitors = api.get_monitors() - monitor = next((m for m in monitors if m.get('name') == monitor_name), None) - - result = { - 'monitor_id': monitor['id'], - 'push_token': monitor['pushToken'], - 'group_name': group_name, - 'group_id': group['id'], - 'monitor_name': monitor_name - } - print(json.dumps(result)) - - api.disconnect() - - if __name__ == '__main__': - main() - mode: '0755' - delegate_to: localhost - become: no - - - name: Run Uptime Kuma UPS monitor setup script - command: > - {{ ansible_playbook_python }} - /tmp/setup_uptime_kuma_ups_monitor.py - "{{ uptime_kuma_api_url }}" - "{{ uptime_kuma_username }}" - "{{ uptime_kuma_password }}" - "{{ uptime_kuma_monitor_group }}" - "{{ monitor_name }}" - "{{ monitor_friendly_name }} - Alerts when UPS goes on battery or loses communication" - "{{ ups_heartbeat_timeout_seconds }}" - "{{ ups_heartbeat_retries }}" - "{{ ntfy_topic }}" - register: monitor_setup_result - delegate_to: localhost - become: no - changed_when: false - - - name: Parse monitor setup result - set_fact: - monitor_info_parsed: "{{ monitor_setup_result.stdout | from_json }}" - - - name: Set push URL as fact - set_fact: - uptime_kuma_ups_push_url: "{{ uptime_kuma_api_url }}/api/push/{{ monitor_info_parsed.push_token }}" - - name: Install required packages for UPS monitoring package: name: @@ -424,107 +283,25 @@ mode: '0755' - name: Create UPS heartbeat monitoring script - copy: + template: dest: "{{ ups_monitoring_script_path }}" - content: | - #!/bin/bash - - # UPS Heartbeat Monitoring Script - # Sends heartbeat to Uptime Kuma only when UPS is on mains power - # When on battery or communication lost, no heartbeat is sent (triggers timeout alert) - - LOG_FILE="{{ ups_log_file }}" - UPTIME_KUMA_URL="{{ uptime_kuma_ups_push_url }}" - UPS_NAME="{{ ups_name }}" - - log_message() { - echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" - } - - send_heartbeat() { - local message="$1" - - local encoded_message - encoded_message=$(printf '%s\n' "$message" | sed 's/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g; s/%/%25/g') - - local response http_code - response=$(curl -s -w "\n%{http_code}" "$UPTIME_KUMA_URL?status=up&msg=$encoded_message" 2>&1) - http_code=$(echo "$response" | tail -n1) - - if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then - log_message "Heartbeat sent: $message (HTTP $http_code)" - return 0 - else - log_message "ERROR: Failed to send heartbeat (HTTP $http_code)" - return 1 - fi - } - - main() { - local status charge runtime load - - status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) - - if [ -z "$status" ]; then - log_message "ERROR: Cannot communicate with UPS - NOT sending heartbeat" - exit 1 - fi - - charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) - runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) - load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) - - if [[ "$status" == *"OL"* ]]; then - local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" - send_heartbeat "$message" - exit 0 - else - log_message "UPS not on mains power (status=$status) - NOT sending heartbeat" - exit 1 - fi - } - - main + src: templates/ups_heartbeat.sh.j2 owner: root group: root mode: '0755' - name: Create systemd service for UPS heartbeat - copy: + template: dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.service" - content: | - [Unit] - Description=UPS Heartbeat Monitor - After=network.target nut-monitor.service - - [Service] - Type=oneshot - ExecStart={{ ups_monitoring_script_path }} - User=root - StandardOutput=journal - StandardError=journal - - [Install] - WantedBy=multi-user.target + src: templates/ups-heartbeat.service.j2 owner: root group: root mode: '0644' - name: Create systemd timer for UPS heartbeat - copy: + template: dest: "/etc/systemd/system/{{ ups_systemd_service_name }}.timer" - content: | - [Unit] - Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds - Requires={{ ups_systemd_service_name }}.service - - [Timer] - OnBootSec=1min - OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec - Persistent=true - - [Install] - WantedBy=timers.target + src: templates/ups-heartbeat.timer.j2 owner: root group: root mode: '0644' @@ -561,22 +338,12 @@ - " Off Delay: {{ ups_offdelay }}s (time after shutdown before UPS cuts power)" - " On Delay: {{ ups_ondelay }}s (time after mains returns before UPS restores power)" - "" - - "Uptime Kuma Monitoring:" - - " Monitor Name: {{ monitor_friendly_name }}" - - " Monitor Group: {{ uptime_kuma_monitor_group }}" - - " Push URL: {{ uptime_kuma_ups_push_url }}" - - " Heartbeat Interval: {{ ups_heartbeat_interval_seconds }}s" - - " Timeout: {{ ups_heartbeat_timeout_seconds }}s" + - "Health reporting:" + - " Check interval: {{ ups_heartbeat_interval_seconds }}s" + - " Push URL: {{ healthcheck_push_url | default('', true) | ternary('set', 'not set - exit code only') }}" - "" - "Scripts and Services:" - " Script: {{ ups_monitoring_script_path }}" - " Log: {{ ups_log_file }}" - " Service: {{ ups_systemd_service_name }}.service" - " Timer: {{ ups_systemd_service_name }}.timer" - - - name: Clean up temporary Uptime Kuma setup script - file: - path: /tmp/setup_uptime_kuma_ups_monitor.py - state: absent - delegate_to: localhost - become: no diff --git a/ansible/infra/nodito/nodito_secrets.yml b/ansible/infra/nodito/nodito_secrets.yml deleted file mode 100644 index c34b1e8..0000000 --- a/ansible/infra/nodito/nodito_secrets.yml +++ /dev/null @@ -1,12 +0,0 @@ -$ANSIBLE_VAULT;1.1;AES256 -39386134336137333139343861353965353665626161303634323563386563356364373265616163 -3237623438636161323366313363356335613566303338630a663732316663333136653737663935 -34373466316538663962636134303530323837316563336136653636643430363566376363323666 -3531613636393331310a306633613030343138656237663061653438336562323135633732356563 -33396136656564353465636361613161326465303166373131343562363338303966666362643532 -33383561366539393930633766363533313363653733393263376634316362643863376635366638 -36396333633231616662323465323932656565396138313264613832346261616231333265636439 -61666334613662373839613833613663333436373365376534643662656335316536303739616437 -65333635613330353536353162646234323266316338643435653864666364313734386665303830 -34383961633734646134373866623330663038366130306265656466653562643764346162313333 -353664363335303433393330306332666336 diff --git a/ansible/infra/nodito/nodito_vars.yml b/ansible/infra/nodito/nodito_vars.yml deleted file mode 100644 index c0002f3..0000000 --- a/ansible/infra/nodito/nodito_vars.yml +++ /dev/null @@ -1,28 +0,0 @@ -# Nodito CPU Temperature Monitoring Configuration - -# Temperature Monitoring Configuration -temp_threshold_celsius: 80 -temp_check_interval_minutes: 1 - -# Script Configuration -monitoring_script_dir: /opt/nodito-monitoring -monitoring_script_path: "{{ monitoring_script_dir }}/cpu_temp_monitor.sh" -log_file: "{{ monitoring_script_dir }}/cpu_temp_monitor.log" - -# System Configuration -systemd_service_name: nodito-cpu-temp-monitor - -# ZFS Pool Configuration -zfs_pool_name: "proxmox-tank-1" -zfs_disk_1: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN0Z" # First disk for RAID 1 mirror -zfs_disk_2: "/dev/disk/by-id/ata-ST4000NT001-3M2101_WX11TN2P" # Second disk for RAID 1 mirror -zfs_pool_mountpoint: "/var/lib/vz" - -# UPS Configuration (CyberPower CP900EPFCLCD via USB) -ups_name: cyberpower -ups_desc: "CyberPower CP900EPFCLCD" -ups_driver: usbhid-ups -ups_port: auto -ups_user: counterweight -ups_offdelay: 120 # Seconds after shutdown before UPS cuts outlet power -ups_ondelay: 30 # Seconds after mains returns before UPS restores outlet power diff --git a/ansible/infra/nodito/templates/nut.conf.j2 b/ansible/infra/nodito/templates/nut.conf.j2 new file mode 100644 index 0000000..1f8a72f --- /dev/null +++ b/ansible/infra/nodito/templates/nut.conf.j2 @@ -0,0 +1,2 @@ +# Managed by Ansible +MODE=standalone diff --git a/ansible/infra/nodito/templates/ups-heartbeat.service.j2 b/ansible/infra/nodito/templates/ups-heartbeat.service.j2 new file mode 100644 index 0000000..71484c1 --- /dev/null +++ b/ansible/infra/nodito/templates/ups-heartbeat.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=UPS Heartbeat Monitor +After=network.target nut-monitor.service + +[Service] +Type=oneshot +ExecStart={{ ups_monitoring_script_path }} +User=root +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/infra/nodito/templates/ups-heartbeat.timer.j2 b/ansible/infra/nodito/templates/ups-heartbeat.timer.j2 new file mode 100644 index 0000000..925998e --- /dev/null +++ b/ansible/infra/nodito/templates/ups-heartbeat.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run UPS Heartbeat Monitor every {{ ups_heartbeat_interval_seconds }} seconds +Requires={{ ups_systemd_service_name }}.service + +[Timer] +OnBootSec=1min +OnUnitActiveSec={{ ups_heartbeat_interval_seconds }}sec +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/infra/nodito/templates/ups.conf.j2 b/ansible/infra/nodito/templates/ups.conf.j2 new file mode 100644 index 0000000..0fa0a60 --- /dev/null +++ b/ansible/infra/nodito/templates/ups.conf.j2 @@ -0,0 +1,9 @@ +# Managed by Ansible +maxretry = 3 + +[{{ ups_name }}] + driver = {{ ups_driver }} + port = {{ ups_port }} + desc = "{{ ups_desc }}" + offdelay = {{ ups_offdelay }} + ondelay = {{ ups_ondelay }} diff --git a/ansible/infra/nodito/templates/ups_heartbeat.sh.j2 b/ansible/infra/nodito/templates/ups_heartbeat.sh.j2 new file mode 100644 index 0000000..c8d5d0d --- /dev/null +++ b/ansible/infra/nodito/templates/ups_heartbeat.sh.j2 @@ -0,0 +1,68 @@ +#!/bin/bash + +# UPS heartbeat check - managed by Ansible (infra/nodito/34_nut_ups_setup_playbook.yml) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed {{ ups_systemd_service_name }}.service +# Reporting anywhere else is optional and generic. + +LOG_FILE="{{ ups_log_file }}" +UPS_NAME="{{ ups_name }}" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" + +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +report() { + local status="$1" + local message="$2" + + # No push URL is normal, not an error: the exit code below is still a + # complete answer for anything reading unit state. + [ -n "$PUSH_URL" ] || return 0 + + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/%/%25/g; s/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + + local response http_code + response=$(curl -s --max-time 10 --retry 2 -w "\n%{http_code}" "${PUSH_URL}?status=${status}&msg=${encoded_message}&ping=" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Reported ${status}: $message (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to report ${status} (HTTP $http_code)" + return 1 + fi +} + +main() { + local status charge runtime load + + status=$(upsc ${UPS_NAME}@localhost ups.status 2>/dev/null) + + if [ -z "$status" ]; then + log_message "ERROR: Cannot communicate with UPS" + report "down" "cannot communicate with UPS ${UPS_NAME}" + exit 1 + fi + + charge=$(upsc ${UPS_NAME}@localhost battery.charge 2>/dev/null) + runtime=$(upsc ${UPS_NAME}@localhost battery.runtime 2>/dev/null) + load=$(upsc ${UPS_NAME}@localhost ups.load 2>/dev/null) + + if [[ "$status" == *"OL"* ]]; then + local message="UPS on mains (charge=${charge}% runtime=${runtime}s load=${load}%)" + log_message "$message" + report "up" "$message" + exit 0 + else + log_message "UPS not on mains power (status=$status)" + report "down" "UPS not on mains (status=${status} charge=${charge}%)" + exit 1 + fi +} + +main diff --git a/ansible/infra/nodito/templates/upsd.conf.j2 b/ansible/infra/nodito/templates/upsd.conf.j2 new file mode 100644 index 0000000..baddf21 --- /dev/null +++ b/ansible/infra/nodito/templates/upsd.conf.j2 @@ -0,0 +1,2 @@ +# Managed by Ansible +LISTEN 127.0.0.1 3493 diff --git a/ansible/infra/nodito/templates/upsd.users.j2 b/ansible/infra/nodito/templates/upsd.users.j2 new file mode 100644 index 0000000..8d43784 --- /dev/null +++ b/ansible/infra/nodito/templates/upsd.users.j2 @@ -0,0 +1,4 @@ +# Managed by Ansible +[{{ ups_user }}] + password = {{ ups_password }} + upsmon master diff --git a/ansible/infra/nodito/templates/upsmon.conf.j2 b/ansible/infra/nodito/templates/upsmon.conf.j2 new file mode 100644 index 0000000..dc301e1 --- /dev/null +++ b/ansible/infra/nodito/templates/upsmon.conf.j2 @@ -0,0 +1,34 @@ +# Managed by Ansible +MONITOR {{ ups_name }}@localhost 1 {{ ups_user }} {{ ups_password }} master + +MINSUPPLIES 1 +SHUTDOWNCMD "/sbin/shutdown -h +0" +POLLFREQ 5 +POLLFREQALERT 5 +HOSTSYNC 15 +DEADTIME 15 +POWERDOWNFLAG "/etc/killpower" +OFFDURATION 30 +RBWARNTIME 43200 +NOCOMMWARNTIME 300 +FINALDELAY 5 + +# Notifications +NOTIFYMSG ONLINE "UPS %s on line power" +NOTIFYMSG ONBATT "UPS %s on battery" +NOTIFYMSG LOWBATT "UPS %s battery is low" +NOTIFYMSG FSD "UPS %s: forced shutdown in progress" +NOTIFYMSG COMMOK "Communications with UPS %s established" +NOTIFYMSG COMMBAD "Communications with UPS %s lost" +NOTIFYMSG SHUTDOWN "Auto logout and shutdown proceeding" +NOTIFYMSG REPLBATT "UPS %s battery needs replacing" + +# Log all events to syslog +NOTIFYFLAG ONLINE SYSLOG +NOTIFYFLAG ONBATT SYSLOG +NOTIFYFLAG LOWBATT SYSLOG +NOTIFYFLAG FSD SYSLOG +NOTIFYFLAG COMMOK SYSLOG +NOTIFYFLAG COMMBAD SYSLOG +NOTIFYFLAG SHUTDOWN SYSLOG +NOTIFYFLAG REPLBATT SYSLOG diff --git a/ansible/infra/nodito/templates/zfs-health-monitor.service.j2 b/ansible/infra/nodito/templates/zfs-health-monitor.service.j2 new file mode 100644 index 0000000..104d974 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-health-monitor.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=ZFS Pool Health Monitor +After=zfs.target network.target + +[Service] +Type=oneshot +ExecStart={{ zfs_monitoring_script_path }} +User=root +Environment=HEALTHCHECK_PUSH_URL={{ healthcheck_push_url }} +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/infra/nodito/templates/zfs-health-monitor.timer.j2 b/ansible/infra/nodito/templates/zfs-health-monitor.timer.j2 new file mode 100644 index 0000000..7878ee3 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-health-monitor.timer.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Run ZFS Pool Health Monitor daily +Requires={{ zfs_systemd_health_service_name }}.service + +[Timer] +OnBootSec=5min +OnUnitActiveSec={{ zfs_check_interval_seconds }}sec +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 new file mode 100644 index 0000000..04b3514 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-monthly-scrub.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=ZFS Monthly Scrub for {{ zfs_pool_name }} +After=zfs.target + +[Service] +Type=oneshot +ExecStart=/sbin/zpool scrub {{ zfs_pool_name }} +User=root +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 new file mode 100644 index 0000000..36ad3d7 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs-monthly-scrub.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Run ZFS Scrub on last day of every month at 4:00 AM +Requires={{ zfs_systemd_scrub_service_name }}.service + +[Timer] +OnCalendar=*-*~01 04:00:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/ansible/infra/nodito/templates/zfs_health_monitor.sh.j2 b/ansible/infra/nodito/templates/zfs_health_monitor.sh.j2 new file mode 100644 index 0000000..3ff4c41 --- /dev/null +++ b/ansible/infra/nodito/templates/zfs_health_monitor.sh.j2 @@ -0,0 +1,181 @@ +#!/bin/bash + +# ZFS pool health check - managed by Ansible (infra/nodito/32_zfs_pool_setup_playbook.yml) +# +# The exit code is the answer and systemd keeps it: +# systemctl is-failed {{ zfs_systemd_health_service_name }}.service +# Reporting anywhere else is optional and generic. + +LOG_FILE="{{ zfs_log_file }}" +POOL_NAME="{{ zfs_pool_name }}" +PUSH_URL="${HEALTHCHECK_PUSH_URL:-}" +HOSTNAME=$(hostname) + +# Function to log messages +log_message() { + echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "$LOG_FILE" +} + +# Function to check pool health using JSON output +check_pool_health() { + local pool="$1" + local issues_found=0 + + # Get pool status as JSON + local pool_json + pool_json=$(zpool status -j "$pool" 2>&1) + + if [ $? -ne 0 ]; then + log_message "ERROR: Failed to get pool status for $pool" + log_message " -> $pool_json" + return 1 + fi + + # Check 1: Pool state must be ONLINE + local pool_state + pool_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].state') + + if [ "$pool_state" != "ONLINE" ]; then + log_message "ISSUE: Pool state is $pool_state (expected ONLINE)" + issues_found=1 + else + log_message "OK: Pool state is ONLINE" + fi + + # Check 2: Check all vdevs and devices for non-ONLINE states + local bad_states + bad_states=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.state? and .state != "ONLINE") | + "\(.name // "unknown"): \(.state)" + ' 2>/dev/null) + + if [ -n "$bad_states" ]; then + log_message "ISSUE: Found devices not in ONLINE state:" + echo "$bad_states" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: All devices are ONLINE" + fi + + # Check 3: Check for resilvering in progress + local scan_function scan_state + scan_function=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + + if [ "$scan_function" = "RESILVER" ] && [ "$scan_state" = "SCANNING" ]; then + local resilver_progress + resilver_progress=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.issued // "unknown"') + log_message "ISSUE: Pool is currently resilvering (disk reconstruction in progress) - ${resilver_progress} processed" + issues_found=1 + fi + + # Check 4: Check for read/write/checksum errors on all devices + # Note: ZFS JSON output has error counts as strings, so convert to numbers for comparison + local devices_with_errors + devices_with_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" ' + .pools[$pool].vdevs[] | + .. | objects | + select(.name? and ((.read_errors // "0" | tonumber) > 0 or (.write_errors // "0" | tonumber) > 0 or (.checksum_errors // "0" | tonumber) > 0)) | + "\(.name): read=\(.read_errors // 0) write=\(.write_errors // 0) cksum=\(.checksum_errors // 0)" + ' 2>/dev/null) + + if [ -n "$devices_with_errors" ]; then + log_message "ISSUE: Found devices with I/O errors:" + echo "$devices_with_errors" | while read -r line; do + log_message " -> $line" + done + issues_found=1 + else + log_message "OK: No read/write/checksum errors detected" + fi + + # Check 5: Check for scan errors (from last scrub/resilver) + local scan_errors + scan_errors=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.errors // "0"') + + if [ "$scan_errors" != "0" ] && [ "$scan_errors" != "null" ] && [ -n "$scan_errors" ]; then + log_message "ISSUE: Last scan reported $scan_errors errors" + issues_found=1 + else + log_message "OK: No scan errors" + fi + + return $issues_found +} + +# Function to get last scrub info for status message +get_scrub_info() { + local pool="$1" + local pool_json + pool_json=$(zpool status -j "$pool" 2>/dev/null) + + local scan_func scan_state scan_start + scan_func=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.function // "NONE"') + scan_state=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.state // "NONE"') + scan_start=$(echo "$pool_json" | jq -r --arg pool "$pool" '.pools[$pool].scan_stats.start_time // ""') + + if [ "$scan_func" = "SCRUB" ] && [ "$scan_state" = "SCANNING" ]; then + echo "scrub in progress (started $scan_start)" + elif [ "$scan_func" = "SCRUB" ] && [ -n "$scan_start" ]; then + echo "last scrub: $scan_start" + else + echo "no scrub history" + fi +} + +# Optional reporting to whatever is watching. No push URL is normal, not an +# error: the script's exit code is still a complete answer for anything reading +# unit state. +report() { + local status="$1" + local message="$2" + + [ -n "$PUSH_URL" ] || return 0 + + log_message "Reporting ${status}: $message" + + # URL encode the message + local encoded_message + encoded_message=$(printf '%s\n' "$message" | sed 's/%/%25/g; s/ /%20/g; s/(/%28/g; s/)/%29/g; s/:/%3A/g; s/\//%2F/g') + + local response http_code + response=$(curl -s --max-time 10 --retry 2 -w "\n%{http_code}" "${PUSH_URL}?status=${status}&msg=${encoded_message}&ping=" 2>&1) + http_code=$(echo "$response" | tail -n1) + + if [ "$http_code" = "200" ] || [ "$http_code" = "201" ]; then + log_message "Report sent successfully (HTTP $http_code)" + return 0 + else + log_message "ERROR: Failed to report (HTTP $http_code)" + return 1 + fi +} + +# Main health check logic +main() { + log_message "==========================================" + log_message "Starting ZFS health check for pool: $POOL_NAME on $HOSTNAME" + + # Run all health checks + if check_pool_health "$POOL_NAME"; then + local scrub_info + scrub_info=$(get_scrub_info "$POOL_NAME") + + local message="Pool $POOL_NAME healthy ($scrub_info)" + report "up" "$message" + + log_message "Health check completed: ALL OK" + exit 0 + else + log_message "Health check completed: ISSUES DETECTED" + report "down" "Pool $POOL_NAME unhealthy - see $LOG_FILE" + exit 1 + fi +} + +# Run main function +main diff --git a/ansible/infra_secrets.yml b/ansible/infra_secrets.yml index f7fa9d5..567c768 100644 --- a/ansible/infra_secrets.yml +++ b/ansible/infra_secrets.yml @@ -1,191 +1,194 @@ $ANSIBLE_VAULT;1.1;AES256 -35396230336432656135363562623433303661376462666131373963633839386238636162613065 -6563613965386631663235303662636261323635393031620a633032313564306337666631646130 -32623664343966653866643438613965303936616234313036303061336139633535653333636439 -3432666530356639640a353631666335653863326232323061663136646332316535656437336235 -31623433376561633931626537333162323331336338326332633630343530656364323536303336 -62383763643033383138333365336130356264626661663732313561323465366437323164393733 -38333532376239373861613566333466316234623564316434383434393064323330663438393139 -33323263323038376135623439376132663637653634613465653865623163366365633861336161 -31323337656463396464303963353236383638646438646531393934656137303063343238363331 -61656262313037306531343437306361646637373864356662386333346134626637346363383931 -39386565643335633863396662363264633432646336383235303466616262663533303965623831 -39333563353431636438646639376663666439343161396163656463366333373362313466626465 -36376665653561636135383531336236643461303261393534373331633934346438323035663637 -63343634376532386532633435616661656336313966363566323937653533613665363461306638 -65393366346662356264353433386534326537363436313836373735616663303131643063313064 -61396634623262616661333365313466636332653261666539653135656166383933373436333032 -64393634396531656536323436373061626531663436313630373865323838376639386430376331 -64306138653931383335366130386334386236363062363631333465303635303531386261383662 -65343236663334356631306439613263653162626536303163343463353431363766643636653166 -33613439643634323331323165386337316463383232646232616439306565353866646635353163 -30343965316234666164633863333632376561383638323639353361313263616561613335316135 -62326466623134313236316430303861356335346633333337373639343766393639353431653061 -39363266663732393434326539353032666161623265633464626237613364396239626163343466 -36303364326638346230326632333636643866663937373864326161663765363535333761373138 -36333034623532616261613336326134626334313063353165303035383166303733393939303231 -30643731353963303839623265336663663436643839666366656661313232623036643232646465 -33666539343565363464373530323033353330343139666239623663646166373733363736633335 -61316134646234653638373639356665396663376366366366383831643938633933656261316431 -65346334653864663863333239383131326665316234363536356234373266653064353562623934 -61656632343235376235323466316562336337326637386534393436643664396634326261643736 -65376264376430333461353336656539636133343933323966323431376638653161316535323862 -65383661373763336638333461303433373365633161623333326535343932366537633362663431 -38663630336139313831656634653736633162353038333438643533633962633166633330333939 -38636266316162303039303737666265633230373364316362393562383532393539376133633137 -31366562666337313866343662363732333932643866646164633964323435393139633766376165 -39616262383366376164383334353633313431393830663962356165633734306337666330386235 -61303664303231346133353130666132623533343938613962363334663466313833633430393337 -38393631326534316363333966366138313836303830393763376633356234353335353366323962 -66366362386338353830346232303032666265343138313836633539333763626132653065656136 -35653634616638346138613464393464636239633762346236613030663766363839373134323730 -34373936323838633035373764373461636533333033646439386665653539653066616135333664 -38643964613966623833653233343464333464633134353431396536656432353738333761363932 -35646436323565656463613433353563373936316130373565636638333237373038613435633765 -63393538613531323063383434643739383134643863313739343863396437333461343936653666 -30656161656638316534633264626338666431333035633432346666636235343438303936363738 -61633838303238346336316365623062356464393463386237366664656134626231633831313964 -39393663626337366264663462663532636365623834386337333538633833633337646138373331 -30633639313633643233386664643033386136306265393233313236343166373838393165316234 -36366339373730343933343661353462656165643361626335663661343666313762613534313934 -37333764363565316163656334613631633938646564323535336164396438626261346365306233 -39383338303063343839633161623631343165336466383838633935323466613831313137313732 -34303961306638363233346436363432363366666162326438623962653138643661336233303464 -38626464326331623735313335356637376633373065366235303834343661633866303965646236 -34373066323464396363663562363239343261643930306632356566656364373231323039316339 -63656630356566343131363935396630313632383137323263363936383862623534623532636563 -61616236346237373031343034646231663833613132356362303532633062356638383266303438 -38303937376166663333653766633165306631366333323831333534613438363063323839633263 -32363132626335656434303130386330626530346463616136393137333863393965346639366261 -35313166356437393163663866626437383633323031373931643439623866353162646566303739 -61373530353264663436643937323431393762623933383233383833663235656434356639353664 -31336230666464396533633735313632663064333436306235303334653064303237303737636132 -62663461383761303863663538626262313038343739656262306564653034353031383234666534 -39643765393634323937656263626463373865643561376235663131306431306564363463346266 -61313835623933336436636131316130346337623835366366343735613562616137643466616631 -65386234656235383639353432333663626538303564303065383064636638333466366663346535 -62373034353430393733656163303933393332373030646630646638373861393463663434303264 -31626439663239373732336330356131613862343836363232326663643436643264323531366530 -62346532633565373961666464626431386431623038313464396361643934646332373735383339 -31343333623739623437356431623439333134333932363734663734636239623930663234656562 -63633561646362643738306632343732636166303636383939626135356536356134656262356636 -36346338643433393462303162343737643330656230643339386536393738353161303831663338 -65346262346136363565333261656164356664323533626537363562313935653766623535313361 -64343637376530333833343935633338303738376664323363643131383938333636636365333134 -33306363396233613164363461616238616364633461326666343366343534663166646439316437 -30396236333236393466323639663930636333663261373563346364613366616336643335646139 -39613632663131323432663938373537663335643934616333656662346432353766326631643034 -35316436653765386235366464373565386463333137343039653732386235633162373834383730 -66323139363137333835613132383364333437643332613930313463343539393736333336363039 -37333136373933353932353637663238363336613133626237393436353762316463633437626130 -33373762643736643465353030653538653262363531643935363836396631653431663231373636 -33656637336535646538383364393165313461616565316330653362643037333635386433626566 -64346166393130636661313731343436383139663831323031346663613761363637353537373632 -37316135333930386632613039356465353761383433333465333033646433373235323932343333 -39643532306133383665643838303366613631386439303862613963656434616461316236383233 -39356665383463356435646630313932626263643037623038613662383536663831316537613038 -63336262346361326664333432386465376634383534373562643238323462633334626366343761 -65303264316332656663616161373335313863396363643462656661613766306632636138623234 -35623666313130633836366161623862333161376464326166616539633835613861343038363736 -62646661393837353266663962313836326162306665323230353764633965336634373934366230 -31633364373164653163626162353934333933303332623564653362396665303961393039343565 -31386632363536373561643838613465633462346539343266613562393236626166383332653335 -38633432333535633532363239653332386565626232653963653930623735633632356264653463 -38376238393961653834616134376462363862633462333265663932653338656362363461613533 -33666137333639616237306263366264623236343732393134643266353331323634343462613438 -64616464316535363766666331373638313566643132316131336337393131346334393632633366 -32633964616461366435643234326234366434363131643437346131396139323331626237396365 -61666437643335396432373932653066663862623035646365343561616630663835663161303537 -63653134303134633861393036653464613265646135353061343663623964333436663834396336 -30313063336131626534396438393731616233333365393965383863323461303561356332363763 -66313863373164663734663462393431653864623764656536366266643231313765363430393134 -33306565646532306639346635323461313736663533303266643433306634326565383232643939 -65623738643739353363653266353534323332363037373834366135643537313739353263636638 -32653234353538386461323632613536366434613262653538333661313333363539393664363363 -64656461356661613161383562653465353866623961653232386265613864343562356639336339 -30653131623130633332343931343537326636356134303336306139333736303763303533323735 -37636131323530336264356264383263343038316264646634356338353761383033646631396364 -62303330386362623436343161353731623435356331373666613335653761653366356433373564 -34623062613439376432303765333763346662393866326337613061616138396430326133376139 -34663337393938343563363266386536616334623964633132303361393137623936623434393562 -37383038666166643537316439323333373431383661643863653034323330366565306630643161 -34643931373966306539623361323261633061363837643538373830343730646534366435636335 -35303862396563333835636431636433393933363533643636343265613638343933373438323563 -64613937666330343038343437616131303630323961306135386264363361383163393836353963 -35303064333065373533366535366361393436393535343965333865646137366166376530346436 -62386432316264663532333465623265313734333337323038613066663533356638666662323334 -35353165633564666130353962386439373863353535373236653338646633356531373230373262 -38666637663661396538333031653966616365353961653530613465623362393932643064323035 -39636534663464363064323862336633663038393834663333353235353562363934373739656664 -65653362346632613862393133316362366665646130326232313138323237646664383565366466 -34666231346433656235363362313434313438396361383231656135306330303035393530393530 -61643331626464646266326239336565613562353462323834393731326339383461313164633930 -31653633373633333539373962383866663739653135336463623837643165323831366562326536 -63613233383533636338646466363962313933393035356339316162616436306138383262633430 -37356366636662363231366532393763633564366461383162613539633734626563303864316233 -61653339393465626631373664306165353964373966623733386532636233346535326132366261 -34663036343665316234303439313064643939663833386633366264653163353234356237343837 -39613134623138616264613732383065623434303336393866633836306432663562393730343833 -35633139343130343236653931346266613665616230633032626665663031313262306530396364 -30663863326632633261346363633838653339393239376462386631303865636364616238646663 -37663033363433333531353332383866383065363337643265636264353562663635393766303265 -39323832663261653831616561333639373838316138356333646439356535333531626530633238 -33623039666433633436363365666431623039353065386564653139336138346136333238663866 -63356237373130343163303936643562373037613361663834386562353337663061313934643430 -32646137636139646263336439663730393737613061663933386165343034386161343333623834 -33323163626266363136376536616362636631353264363736663366636266653238656362633934 -30373664643933613637346363343964346463363036313961633961303832323834376337666630 -38363862613136396663626436303237396435333464303938303562333034613532373134353233 -38653436336461386139303864393338656661653137646137336332616435656638393231323838 -62663430663330653963313939663362656430303761656633343837306265366332393030363930 -30346439393837363333636136353235353463333563656332633236663033633835363531633261 -61643632396666663934346139373235346261626339386431333064333632333034333031666439 -66336561303536656462343363653136616562306362333662643938616437663938393933633463 -38636339356565663532663235326266386332353764656639363165336634653639336463616362 -66393131656631646133376461343438386162306430613561323630613865626235303438346233 -33373839323732666437363333346336663963323066353561396338383633626138323337383032 -35663132623364666537343036303136336333633864323834633265343936366631363865333962 -63306433333035363565303866333234363038663934366361656539333963373932306535313638 -61626661336330393265353064616263396533316361616534343566613833363061346230333036 -62626130383630306361646531666436613461333865343131626465323361373163336539343163 -39313036373236646162333563393561646663326631613333646238626463396364343061383233 -31373365396535326131616237323636623364633263386564613763623636623138386430376364 -39326637386237616332363531393534636262393336366335306665636466346134633365386665 -33646266386635636361383937616630376530643237396366666231313930373939313566383938 -64646436613039633835363966343838656461313834643438356166356566663861663833666137 -32666437626637323766303962303164393565626362656430353439383839313166303966366433 -64383730336566633336303966376366343332633366373662353337663965616333363033393765 -36306536316135303164306164313239333938646630633864336237346161303963376531373464 -34373465366231336365323030376235306662313433616433633962363564326262363631666136 -65613538656534366136303030303339383136323137633934633662646537353235646534313332 -31366365613533653339646131653239343663346239613934393666373536386635656534376165 -64613638643338613738353263633664636164353265616536626132346663636133393137616433 -33663635343063373465303936313731333765396137393235643566656332666634636462386461 -37323065373733333433316463663635653630656131313235386663306134373364393765356239 -38393964346361356135626331653161633536333463666463653936656531313463666632393061 -36383166633935343063663763663536326466646663383763333161613165393665383030396262 -34666631316262616131346233613035343236613838333036663661386464346136383737323036 -66366632313130646430616563653731346362343339363235616335393964633335326633376135 -34386666353864303161346263663265393731326332303231323934383838343934376339373139 -34663666363335663165636237303330306338393136653339633733663364623461303134303839 -62366337396363613737623138666565613530643861653830616332656435373233656266353461 -66616636393864643338616164316131623464613466396562386238663364623630303338643837 -64633433666139333864313862663832306230663339643933663530383939396230393032323861 -30376464336566303833306664613633333335326562396462326437343437323032363065363237 -61646238363436383134343434353534326537666431333430343065633035353437616366383039 -36616362353536396436623835656462353231653838336666636335666333653130633330353232 -64643733393034343139373733623365333961343139353532313230613134346365303061616663 -36636465353234316361376261303834333532623336646335306636353965613361366430643436 -33306135633538653034643361323235353566636661363436326133326238633964646263376166 -32333862623239333661393930346638663461633762653332386631616666366333333863336239 -63356265663437343037396464396365323336313038336636636562343262396466353333313765 -33396636396562323338383561616366323832616566323464653035366432653833633831633234 -30636237616332383434623338616639633732643134383637343162353234316639333733613033 -31306362623163373631633965623631643537393662626538663966373765383530343064313165 -66303337656537343539323565663636333139383033393462326331643234616138323236663630 -62653163393733613838633739393439343464653738616161663330633236626234313034346462 -31633539303139306361613839366234643535383961346130363433633033373434633565613063 -35323839386463383438663131303665386130653332366463363330363530653533313661363361 -63313539653239303861356638303236396138326131626564666337626332383462633732373465 -36353966383036663733333336643633666633346233363732633837656334316331313032323064 -3739636230343038353664353163363939363465326433613561 +36353131353730316436313965366263666236396164323163376431623232313961323863633035 +6338333965643036666538303864376463623434326564660a396239633264353034653963366466 +66353130376366663730626331346235373935306434303462663763613763663131613335633430 +6335633535333166330a366232306230303033663931333734353636656132383034616336653235 +35636234306134376666666538386561663561626262396135313632393831363665646130643261 +65663137613530393162333838313962393934396135613064333564303666353634396266316135 +65346363663232626261643933353239663635343137646431623561653963363131303033623764 +34616339343835353765363962396665636335623066303062626463343733643137626265643366 +34653139616163666131333330376132343363306132653163343934353034353932383066373462 +31316664383938633361373839353762393838386636393564633430346334323734383430386663 +39313938383830373062336335303135396334323330316537303965653331323130323866663866 +38323462643633396531386632613961343835313931613933373963326465333836323935613436 +63346562313239623162333038663535656633323037363532373536616161303833626539366634 +35363462663539353366623732626237326465663436356630626335623334356465646439376461 +63653230343333613932646431383238373039653461306365326234303365373234613038343338 +30353937353734633631313765346662616138383462353864656264653534653239383936616437 +34313531383438613465626437313130326562316430366234326230303133356631363534396366 +64313732303837666537383161373865653162303034376533636264656564316530613738353435 +38353031616538346438336563336134643261366331623562663534643966306164306132646332 +31336539663131646232353633326637306637353239616331356337353761313763643933303838 +61636333663137623233323561306665666666363731383438373664363538613762393130646261 +66346335373437326536356330613962386662383031366536373635353438316664613362633966 +35393331613634653462323739343466313735376463643035326336396435656364356134626535 +32623036383737623631623332666264653964383939316465353765373132383230336334353463 +38303136356630653561333363306136363230383238666663373130663763336532323631663637 +32633263633835326662366237613639313431366433363730666666333961323036363533613236 +39343963613239333731336461393937633463303530643333626662316563313165613564386663 +34373636383231636461653862356332353161366331633036383837353434373865363139386235 +30323130343534386531666333343765373265626530373335633766353562356562326239336564 +62313033376330383563633138623838633132343130343262663030316435393835313762313564 +30393366393161663265303439383937333739396661393862356132343537616333623035653566 +64316333323733336135383637333739393931333461653731383863616138613234316161623765 +30343964623939663339633139383866336163613933363231303837646565626262303330363338 +64353931623366326666386637363866383761666232663866666438313535393434373933653139 +64636236306632613930363935353736393065353634653235303534316463653438646162663866 +63663839346162396262366563613131356161333137623966653039623436373466663163313938 +64636362383532303136316163306236333134346164376362363730363030643436356631306639 +31376131653162343338613830376466356363356238343436366232393062336433623238323864 +32376133346264383736396166343539363361623864613364613030623936396634616431386263 +35323632353334643466383933356236383432323162386532613535343932353531373065633732 +36626361313436613036363662376337613237353532366163656236323937316437613633383535 +39306334656532383063306332636538666462353836633366353932623962396131663532343364 +64393035303265336638343733366236373466343832333038633138306535303034386134653032 +62633532353462613664636333373461356434666130376332373762653966386238346463333939 +32386233326264666234353139643837636663633362626163356633396465643030383639663538 +31626331393161653038613438613735643364326565633362386331646231333539626133633739 +31336236663863376166636632363965343765666335633464613937653738313230356438313436 +39373936663363626337626164383237626164396337613430373035373339616266666264346164 +66386238373664386138636536616266353466666363383131616532323430663761633139626164 +33323936356565303332383463656533633935373564353461626139343265336334363433333138 +64363534363836666338366339636435366334383339663463376164613866333839366264666237 +35306231656536613138353663626166303366373761323866666436356438653764353431383430 +33323766643431303337323265646631373434393436333335666366306138356337343163336362 +39323131653734366561613864383266386131626164363361656534386363613565336532666162 +38333031656330306635313032343861653037323037616564323537666261316438303631616530 +35303238366338376363383764633865326530643736306634363436643330376432396535363932 +39646666333234326261373835646336663138366237346562396238326530356533326363373664 +38303634383036323232363934626562663864646239653066326565623635653535623137663065 +65376633356136333938623763396138376630636636376434333463313833386435633365623036 +30373937353433396264643231323832373963396437363562623639353764623063363733366165 +38613833353231333836373637626133323965316665313837653335386534326434363034653630 +38323335666466306665663262343737636564393934373338623130636362383564376434626533 +35623761396465333730626563633734373131373132343262653333343439623433313964623463 +33613639616636393563663331326535346131663832313861353434313965393164643538643437 +36363130363934623333666330353236386530626536616333303131396265386263363462356364 +38393864623461643831333731656566636532643338306361666335316265396430376432393730 +34633638636466313232376363306333356564393038323130313634386162333639366162623437 +66306234346430346537343338343064623131366337356262653130616464646336353136336234 +34623063336332666138303961333332326632326137366664306263666538616361316363323163 +38613431353762653139636365353031343465366138396161366538393833666236656264623132 +63313630633231613438613865613333343066376261326333623864636430643866656132353538 +65653731626464323930383636663364313437313531646462623163646630646664356335303764 +61313462623234303961356363386464333061306239643263333135653538313338383134343236 +63396461346438373830643031336165376563636530353836626436613561313536623130613663 +63383939663535326236313736613630313061323837346339353834313434393437383237633534 +65363334643238373864636530386434636565623131366433313562623933366633396565646565 +33626433373834633638656365363261643866343961663566333761306138383865316536303664 +35653463633336616438373732303938393032663961653262333134353762333433373039636638 +33366466336133383061373866323231663038373831386139613535376539393165363034303537 +35346638613137616666623362396564346464376133316262646339333936363961316161663332 +38303239356334633137316333363439303935346230613032643564353734643863636264316136 +39663131376638653138623838323739343064623430386166643962633363373335316239663339 +37633934393664346433316633353831373534653437623065363134626661323732656664633639 +32616539323931626130353836303533363232353066343666366263663565623862333062323237 +31393935313133333837626632346461356134376339633239396438336333376534353063633937 +61653230653666623935346134666130373934643438323464346239336430373635396165323738 +36613432356239323039386364643535346261373739363463373839373063346565616466663239 +33353165353631343730636531363565383363623839346434383737313136646330323461633665 +32376631663637353534636262666532643636366237383539346235383832313933336432363734 +35653866643136383934613261633439343831363234616438333530336437326562366536383834 +33343330316337323064386464656436333430643061663665306534386235336437643965393165 +61373038323539316161363731306334633833376532626537643164373638303438326637376630 +35343333343364326538333862643265336166396466373362326162303430383434626535336163 +31666662653261343631366165656131376462333063626134613466636637383936303933346362 +32386162323230666563353838613033393936386535646130613861356563633566393430313661 +37666561643361653630353262653234636232303534393661343834366562353864343638313134 +36656463653166393432646263626530316361643761646265613534643563306161333539633361 +33613830356238623962623739623437646635356334386637633035373262313539386662353732 +35666534663965333836396664316432633137363761623838356430316566383131363530316462 +62636235396136363538653836633133663465386139636339353664666634343238663861613437 +62656435363832623461363662643262323265353066376538306163376438303838373961303239 +63623066656532353933396534623637356362356231336361393534393465633565653132656130 +37363863666136643461383033653936383935333131343565643664386664326463663466633361 +36313638616533623431316361386539383434653866376363383630323632636237373666393561 +31313135303339633763303762653939386663393439646135656537303331366236396230303063 +32336336366533323936633166623434623365633163643461653966383563653533363335666638 +61313764656437613138663038386336336664396539373930373933326234636662653833656435 +61343962373935653238326265663164363561616435363134313634666331636431633133356563 +37326133363762383266353730343537373934653634366336316635653063313461303237633164 +61333231353135343666343461306161626432636331653962613334376439613265653437663333 +34623638313561356464356534343635653265633531323736386335656635303263636663323930 +39373462663438326366363433343735343234383735643363326339626135313564646432656330 +38653836653938383932643032616534363035366530336663653235366230353332636531653138 +63623537353165623037383361643937623466356464313931666430666632633866363232623230 +32353766383432613233313331356432326430383132616263656361613334383936643066326133 +30643935653662376433353039323836613239336265376663616336663264343331373236373237 +64326462626566386563623235633664653665356461316362663462343665303233313035366435 +32623465656138303161396233623661333238306334646663353263383437653461383366366466 +39303061363530373339623435613765313637616634313731666631653161623439323734383735 +65383666366330623338653561353032373231396431353466313335303935386136613566336331 +62326539383631636263353234393866333031643737663064383130313066396461663466623431 +38616364303439393432663035623138353264656635393633646363366538633262306466336564 +35623733613564323233303034636264336464373566303065383438366338643538666638643466 +63343130613461316465626237343235616536613838623930636136333131616437656631376564 +34366535383961656237333030376139353237343636306165646161323732646663376231393832 +61396263393561356438326237633634333533646638383865336164656465383166373236643063 +63643263303132656336333339623539663765333039396364643462396161366562626362343162 +34363936323230373839373734343332663664373634313364383062353130363362623937656139 +32653237643061646531623030646335383463333136366264363133303666663261343631663762 +36346432386465393765353763353265633837623165303634646137646564356237653336666331 +38376463376130353863636334353633313361313239373264333134333232353765346666326665 +65623333396165353334636532303537366566666130343366383964396365333461646566333431 +37303032643731346361393963303061363837373562393866353962333231623763623236323161 +39383538663837326663636166643864393733653764353536303433653563316339663438353838 +37616135373336326536333932636538336465323130626266613930393266633164636439663532 +37633264356661353836346462626538323631613539636631396436646139666538383838323437 +62663561356430336235343439633135326661653031653063363030326132336462336164373232 +31363131326334613361356461353934656364346666663762306161356463386332366562336630 +39326236656335363630363564316134623435343538386462663161396332366639363033383235 +35306434646330636137323939623565663939643161616336386633633133363963383739633434 +32336433663132363239356266653461633033623232346135353032353265343861316638643265 +66303932643235333764323239653430346132666136663133383935613962353235313336306136 +38373938376238313034346237373035383861613930323936663831346538313937343538663737 +62616139663634333636333635636364643966303565656265653634653437633233396138323831 +64313465643233663663303130653133626638313162356133316139333030663530313232626437 +35613439326639393032393962653933376636663934333764393064346465633637613836333933 +33646336653533666631656132623035323963623134653432363133646339346363336263323135 +34356233373965333362326132383762626436333365363731396134376664363465623362316366 +31623361376161393836623732613034643831363838663733386561313961373632643639666635 +37633061303235306437666337306236316331616330396632356266636137313833346366653031 +64366236306139363861323237396232336665633331326139373461353237336432373366333266 +32326565376464383937356562633730383961306535666464383364656137633662333662626366 +30653736646333333963656566326431376361666465363532393765393764633562626232643836 +32626261373835356634633461303664653362346231343030343433376364643664383464636666 +35323738366332306337616563646231663963353135623133613636666363396530643239626566 +35383330663138333631386363643032383161626439633437343934366635656433386163613663 +30623738656535396134363830626338353864356330613131623832363330633064376531303965 +35313036393031383035633035636165313363333564633938306166666639636436353733303662 +66653264356432383166333630646533313736366130666331306537393262363538313030646134 +38306363663932313230346664303531636639656339323062333739303239333861616533666132 +30353332303233323837346234336633646163643137636166633330633464663935653838313161 +37393533646139313236393234313763353533663638363031393764633862363938643838353937 +36613564353366663434633839303036343665313933326531353831396139613330316632613637 +37666432363330326232366664656462313336323866316633396533313638373462386365663332 +63626136396661353739633263363038326630623037353831323930346431666263333431643562 +39373133636465373064613664323335353236326562343966616439646565383934613462623363 +38613962386236356666643038303435376266656165336263653365353537666362616638636639 +34326262303930303237636339666563613663373863666339663135326661663866346264613734 +34353565613832323132343730396535656264376233356162353265623739613332333261393331 +33616230333033613766643264396633343535376461633330633064613662336532613163373962 +61343065373630383838306631633031656566343765333365373932373234313733396165623639 +65313731346632333235303463393039656232653163336161633265376434623466373361333865 +66323664343766306566663335656138323537656563383835653263323039656166613330613733 +38323064306365656561313163306439653937623536616435616431616133643466336362666330 +64303832343739383430626339663532653363626263393237343234666266323933316566343937 +32373335373237653265303638663331363838313961343936616639646438343633326461626363 +30653964646639663836636437653861343064393332306638363864326439626335373663653335 +33653230626337633863356534393466633162323734623135393931363339383163353466633031 +39343435616137373166376538633330386232333666316566336234373933613534666430353336 +33626637346337623132313833343430613832303932323739326537666531356630356137653963 +30623262373032353934353938346263396338336265393937336539386530343062623761646632 +39653035393535356630626164393464663738616239613166613364396539343039306538613230 +38636261633932643965323966346632616264343262643933346163656436326330336564623265 +30646561303563323937393533663066393733353638323332663736353766336664343265613733 +66643631633864386262343232636465613962386538366163613734386635303837636164383465 +35373331363864393934633563623662326632386534323436663634613030646563643035386539 +37323262646365633739626539346638383039643433343837363561303530376434336131353965 +32336133393139626232313933636136646264613338643238646431316566333662353837636232 +32623639636432333134643962313430623533333237343464383135643361336236396533653761 +66383165363539363938653130633630643865636663313665353839646462613334383266663563 +62333035626539383739656565633861363863333563656339353735336330333633633861643130 +35363062306165623861316639336237373434353437626461383931656531306134313264383664 +37353966616136326264376135373532653630393335336665343463323466353162