m TEc@sdZdkZdklZlZlZdkZdkZdkZdk Z dk Z dk l Z dk lZlZeedeZdklZlZlZdklZlZlZlZdklZd klZd k l!Z!l"Z"l#Z#l$Z$d k%l&Z&d k'l(Z(l)Z)l*Z*l+Z+l,Z,l-Z-l.Z.l/Z/d k0l1Z1dk2l3Z3dZ4e i5dZ6dZ7dei8fdYZ9de9fdYZ:de9fdYZ;de9ei<fdYZ=de=fdYZ>de9ei<fdYZ?de9fdYZ@d e(fd!YZAd"eieiBe(fd#YZCd$ZDd%ZEd&ZFd'd(e$d)e$d*d*d+ZGd,ZHd-fd.YZId/eIe(fd0YZJd1eIeiKe(fd2YZLd3eiMfd4YZNd5fd6YZOdS(7sPStorage implementation using a log written to a single file. $Revision: 1.16 $ N(sPicklers Unpicklersloads(s StringType(spacksunpacktfsync(s BaseStoragesConflictResolutions POSException(s UndoErrors POSKeyErrorsMultipleUndoErrorssVersionLockError(s TimeStamp(sLockFile(sp64su64scpsz64(sFileStoragePacker(sFileStorageFormatters DataHeaders TxnHeadersDATA_HDRs DATA_HDR_LENs TRANS_HDRs TRANS_HDR_LENsCorruptedDataError(sBLATHER(sfsIndextFS21sZODB.FileStoragecGs ti||t|dS(N(tloggertcriticaltmessagetdatatCorruptedTransactionError(RR((t;/data/zmath/zope/lib/python/ZODB/FileStorage/FileStorage.pytpanic2stFileStorageErrorcBstZRS(N(t__name__t __module__(((RR 6st PackErrorcBstZRS(N(R R (((RR 9stFileStorageFormatErrorcBstZdZRS(sHInvalid file format The format of the given file is not valid. (R R t__doc__(((RR <s tCorruptedFileStorageErrorcBstZdZRS(sCorrupted file storage.(R R R(((RRBs RcBstZRS(N(R R (((RRFstFileStorageQuotaErrorcBstZdZRS(sFile storage quota exceeded.(R R R(((RRIs tRedundantPackWarningcBstZRS(N(R R (((RRNst TempFormattercBstZdZdZRS(s5Helper class used to read formatted FileStorage data.cCs ||_dS(N(tafiletselft_file(RR((Rt__init__Ts(R R RR(((RRQs t FileStoragecBs-tZeZdZeedddZdZdZdZ dZ dZ dZ d Z d Zd Zd Zd ZdZedZedZdZdZdZdZdZdZdZdZdZdZdZdZ dZ!dZ"dZ#d Z$dZ%d!Z&d"Z'd#Z(d$Z)d%Z*d&Z+d'Z,d(Z-dd)dd*Z.d+Z/d,Z0d-Z1d.Z2d/Z3dd0Z4dd1dd2Z5d3Z6d4Z7ddd5Z8d6Z9d7Z:d8Z;dd9Z<RS(:Ni'c Cs|o$t|_|otdqHn|dj otdn|djodd}n|p?t|d|_ t |dd|_ t |i |_n d|_ ||_tii|||i\}} }}}}} |i|| ||||| d|_|py$t ||odpd |_Wqtj o`}|itijon|iti jo d }nt!i"i#|oqd }qXn|idjoQ|oJt!i"i#|ot!i$|nt |d|_|ii%t&n|i'} | dj o}d |_)| \}} }}|i|| ||||| t,|i||| ||d |d |d |\|_-|_.} nGd|_)t,|i||| ||d |\|_-|_.} |i0t1|i2t3|i4|_2| |_5t6| |_7} t8i8}t6t8i:|d |df}| |joQ| i;|i;} t=i>}| ddjo t=i@}n|d|| n||_Bd|_C|_DdS(Nscan't create a read-only files,time-travel only supported in read-only modesis.locks.tmpsw+btrbsr+bitltidtstartt read_onlyiii<is,%s Database records %d seconds in the future(ERtTrueRt _is_read_onlytcreatet ValueErrortstoptNonetLockFilet file_namet _lock_filetopent_tfileRt_tfmtt _file_namet BaseStorageRt _newIndexestindextvindexttindexttvindextoid2tidttoid2tidttoid2tid_deletet _initIndexRtIOErrortexcterrnotEFBIGtENOENTtostpathtexiststremovetwritetpacked_versiont_restore_indextrt _used_indexRRt read_indext_post_oidttidt _save_indextmaxt_records_before_savetlent_indext_ltidt TimeStampt_tsttimetttgmtimettimeTimetsecondsRtwarningt complainerRtquotat_quotat_oid2tid_nlookupst_oid2tid_nhits(RR#RRR RTR/R+RR?RDR1R,RQRR0R.R4RNRSR-((RR`s|     !  $     +     $    cCs[||_||_||_||_|i |_ |i |_ ||_ ||_||_dS(N(R+RRIR,t_vindexR-t_tindexR.t_tvindextgett _index_gett _vindex_getR/t_oid2tidR0t _toid2tidR1t_toid2tid_delete(RR+R,R-R.R/R0R1((RR2s        cCs t|iS(N(RHRRI(R((Rt__len__scCsthhhhhhfS(N(tfsIndex(R((RR*sicCs|iodSn|id}|d}t|d}t|d}hd|i <d|i <d|i <d |i <}|i||i|iy;yti|Wntj onXti||WnnX|id7_dS( s<Write the database index to a file to support quick startup.Ns.indexs .index_tmptwbiR+tpostoidR,(RRR t index_namettmp_nameR%tftPicklertpRIRBRCRXtinfotdumptflushtcloseR8R;tOSErrortrenamet_saved(RRkRhRgRfRj((RREs(   6   cCsO|id}tii|o+yti|WqKtj oqKXndS(Ns.index(RR RfR8R9R:R;Ro(RRf((Rt _clear_index s  cCs4|i||}|ptid|in|S(s=Sanity check saved index data by reading the last undone trans Basically, we read the last not undone transaction and check to see that the included records are consistent with the index. Any invalid record records or inconsistent object positions cause zero to be returned. sIgnoring index for %sN(Rt _check_sanityR+RdR?RRRR((RR+RdR?((Rt_sanes c Cs|djodSn|iidd|ii|jodSnd}d}d}x||jo|ii|d|ii d}t |}||d}|djodSn|i |}|p |i}n|i|jodSn|idjoq[n|idjodSn||ijodSn||} ||i} | | joq[nx| | jo||jo|i| }| |i| jp|i|jodSn|i|id| jodSn|d 7}| |i} qsW|Sq[WdS( Nidiiiiitus pi(RdRRtseekttellR!Rt max_checkedtcheckedtreadtrstltu64ttlt_read_txn_headerthRDttlentstatust headerlenttendtopost_read_data_headert recordlenttlocR+R[Re( RR+RdRyRRRxR}R{RR((RRssR       ' c Cs|i}|d}yt|d} Wn d SnXt| } y| i }Wn4t i d \}} tid|| d SnX|id}|id}|id}|d jp|d jp |d jod Snt|}t|tp#t|tot|itowt} | i|| }|ipMt|d} t| d } ||d<| i|| i|i Sqn|i!||}|pd Sn||||fS( s-Load database index to support quick startup.s.indexRis%Failed to load database index: %s: %sR+RdR,RciN(#RR R#RfR%RhR!t UnpicklerRjtloadRktsystexc_infoR4terrRRRR[R+RdR,tlongt isinstancetdictRbt_datatnewindextupdateRRiRlRnR>RtRD( RR#RfRdR+RDR,RkR4RRhRjR((RR>RsF   ' 3      cCsx|iit|do|iin|io|iiny|iWntiddt nXdS(NR$sError saving index on close()R( RRRnthasattrR$R&RERterrorR(R((RRns  cCs|id7_|ii|}|dj o|id7_n|id@djo=ti t dt |i|i|id|i|in|S(Niiis/_oid2tid size %s lookups %s hits %s rate %.1f%%f100.0( RRVR^R[RetresultR!RWRtlogtBLATHERRH(RReR((Rt_get_cached_tids   cCs|i|d|dtS(Nttabort(Rt commitVersiontsrct transactionR(RRR((Rt abortVersionscCs|iotin|ot|to t|tptidn||jotidt |n|o|otidn||i j oti ||n|iz|i||||SWd|iXdS(NsInvalid source versions Can't commit to same version: %ss(Internal error, can't abort to a version(RRt POSExceptiont ReadOnlyErrorRRt StringTypetdesttVersionCommitErrortreprRRt _transactiontStorageTransactionErrort _lock_acquiret_commitVersiont _lock_release(RRRRR((RRs '  cCs|i|d} t| }tdt|it|t } |o,t|i|d} dt|}n d} d}|i|ii|i} g}h} x-| o%|i| }|ii|i| jo| |i|i<|i|i|ii|i|i|| |o>| |i|<|iit|i| |t| } n|ii|ot|ip|| |7} d| |i8sH8siBRi2i($RR]Rtsrcpostp64tspostpackRBRHRtz64tmiddletsdt heredeltaR&Rwt_thltheretoidst current_oidsRRRIR[ReRYtappendR<t_tidRZtpnvRthas_keytvprevR`R(RRRRRRRRRRRRRR((RRs> $" !'  cCs|iS(N(RRB(R((RtgetSizescCsZy|i|SWnDtj ot|n'tj otd|fnXdS(Nsinvalid oid %r(RRIRetKeyErrort POSKeyErrort TypeError(RRe((Rt _lookup_poss cCs|iz|i|}|i||}|io=|i|jo-|i||i d}||i dfSn|i o,|i i|i }||i |ifSn-|i||id}||i |ifSWd|iXdS(NiR(RRRReRdRRtversiont_loadBack_implRRRDtplenRRztbackR(RReRRRdR((RtloadExs  cCs|iz|i|}|i||}|io:|i|jo*|i||i d}||i fSn|i o&|i i|i }||i fSn'|i||id}||i fSWd|iXdS(s%Return pickle data and serial number.iN(RRRReRdRRRRRRRDRRRzRR(RReRRRdR((RR s  cCs|iz|i|}xL|i||}|i|joPn|i }|pt |qqW|i o|i ||i dSn|io|ii|iSn|i ||idSWd|iXdS(Nii(RRRReRdRRRDtserialtprevRRRRRRRzRR(RReRRRd((Rt loadSerial4s$    cCs|iz|i|}d}xtox|i||}|i o"|i pdSn|i }q%n|i |joPn|i }|i }|pdSq%q%W|io5|i||i\}}}}||i |fSn |ii|i|i |fSWd|iXdS(N(RRRReRdR!tend_tidRRRRRRDRRRRt_RRzRR(RReRDRRRdRR((Rt loadBeforeJs0        !#cCsH|iz,|i|}|i||}|iSWd|iXdS(N( RRRReRdRRRR(RReRRd((RtmodifiedInVersionns  cCs|iotin||ij oti||n|iz||i jo|i |n|i |d}d}d} |o|i|}|djo\|i||} | io3| i|jot|| in| i} n| i}n||joW|i||||} | djo(tid|d||fd|qo| }qsn|i}||ii|i} | |i!|i(tlasttfirstRRt_pack_is_in_progressRt UndoSearchRRBtfiltertustfinishedtrangetitsearchRtresults(RRRRR!R((RtundoLog1s&        cCsm|iotin||ij oti||n|iz|i|SWd|i XdS(sUndo a transaction, given by transaction_id. Do so by writing new data that reverses the action taken by the transaction. Usually, we can get by with just copying a data pointer, by writing a file position rather than a pickle. Sometimes, we may do conflict resolution, in which case we actually copy new data that results from resolution. N( RRRRRRRRt _txn_undottransaction_idR(RR&R((RtundoNs   cCsti|d}t|djpt|i|d}|i |}|i i ||i i ||i|ifS(Ns ii(tbase64t decodestringR&RDRHR RRRt_txn_undo_writeR-RYRR`RR(RR&R-RRD((RR%escCs|i}x|djo|ii|d|t|iidd}|ii||iit}|d }||jo|Sn|o|ddjoPqq q Wt ddS(Ni'iiRjsInvalid transaction id( RRBRdRRvR|RzRRRRDt stop_at_packR(RRDR+RRdR((RRrs     cCs|i}|i|ii|i} | |ii} |i|}|i djot dn||i }||i}h}h}x||jo|i|} | i|jo|| i=n| |ii| jpt| | |iify:|i| i|| i| i| i\}}}} } Wn"t j o}||| iQiR(RRIR+tminKeytnextReRt oid_as_longRtnext_oidRR!RRRD(RRQR+RRRReRSRD((Rtrecord_iternexts  (=R R RIRRGR!RR2RaR*RqRERrRtRsR>RnRRRRRRRRRRRRRRRRRRRRRRRRRR R R RR$R'R%RR*R6R8R>R?RRLRMRNRORT(((RRWsre     3 ;     1      $ B ) N      " "      N   < % .  F  c' Cs|i} |i}|i}|i} |i}t } |} |}|| }d}x| ||t}#t|#tjoPntt|#\}}"}}$}!}|djoPnt|"}|}&|&|}|}|$|!|}%||%}t||%jot$|n| |||#||t|%}%|&|%}||%}x>||jo0| ||t%}#tt&|#\}} }}}}t|}t%|pd}|om|d|}t|d}| dd||}t0||d} |djo||| data record file offset vindex -- dictionary, oid -> data record offset for version data tindex -- dictionary, oid -> data record offset tindex is cleared before return There are several default arguments that affect the scan or the return values. TODO: document them. start -- the file position at which to start scanning for oids added beyond the ones the passed-in indices know about. The .index file caches the highest ._pos FileStorage knew about when the the .index file was last saved, and that's the intended value to pass in for start; accept the default (and pass empty indices) to recreate the index from scratch maxoid -- ignored (it meant something prior to ZODB 3.2.6; the argument still exists just so the signature of read_index() stayed the same) The file position returned is the position just after the last valid transaction record. The oid returned is the maximum object id in `index`, or z64 if the index is empty. The transaction id is the tid of the last transaction, or ltid if the index is empty. iiiltisis%s truncated at %ss%s time-stamp reduction at %siRs3%s truncated, possibly due to damaged records at %ss ups %s has invalid status, %s, at %sis'%s has invalid transaction header at %ssIt appears that there is invalid data at the end of the file, possibly due to a system crash. %s truncated to recover from bad data at end.Rus,%s has inconsistent transaction length at %ss/%s data record exceeds transaction record at %ss#%s incorrect previous pointer at %ss"%s data records don't add up at %ss2%s redundant transaction length check failed at %sN(:RRzRvRwt file_sizeRtfmtRR tnameR=RR<RRR+R[RURdRDRRRHRRRRRRR}RRRRt _truncateR|trtlRRgR!RR RRRRRR-ReRR,RRRRRtmaxKeytmaxoidR(RRmR+R,R-R RRRqRgRRRoRdRkRvRlRR}RRURRRzRDRRR((RRA5s             ! "                 !        cCs|idd|i}yd}xd||f}tii |o|d7}q(t i d||t |d}|i|t|||||iPq(WWn,t id|dttid nX|i||idS( Niiis%s.tr%ss$Writing truncated data from %s to %sRcs$couldn't write truncated data for %sRsCouldn't save truncated data(RRvRwRkR!RmtonameR8R9R:RRRR%toRdRRnRRRtStorageSystemErrorR(RRmRdR!RsRkRr((RRns,         tIteratorcBs#tZdZdZeZdZRS(sKA General simple iterator that uses the Python for-loop index protocol icCsJ|i}x.||jo |d}|i||_q W||_|iS(Ni(Rt_Iterator__indexR!RQt_Iterator__current(RR!Rv((Rt __getitem__s    (R R RRvR!RwRx(((RRu s RKcBsYtZdZeZeZeedZdZdZ dZ dZ ddZ RS(s9Iterate over the transactions in a FileStorage file. cCst|tot|d}n||_|idtjot|i n|i dd|i |_ d|_ |djpt|tpt|djpt|tpt|o|i|n||_dS(NRiiil(RRRER%RRRzR=R RmRvRwt _file_sizeRBRR!R R t_skip_to_startt_stop(RRRR ((RR!s  $$cCsdS(Ni((R((RRa0scCs|S(N(R(R((RRL<scCs1|i}|dj od|_|indS(N(RRRR!Rn(RR((RRn?s   c CsI|i}|i}|i} x'| |i|d}t|djodSntd|\} }| |jodSnt |}y|i|d7_Wn-tj o!t|i|d|_nXtoj| |idd|d}||jo9|id}td|i|t |t |q@qqWdS(Niis>8s8siis7%s has inconsistent transaction length at %s (%s != %s)(RRRRzRvRBRRHRRDRZRR|R}t OverflowErrorRt __debug__RoRwRdRRm( RRRZRzRRdR}RRoRDRv((RRzEs0         ic Cs0|idjotdn|i}xy|i|}Wn'tj o}|i pPnnX|i |i jot id|ii|n|i |_ |idj o#|i |ijot|n|idjot|n||id|ijot id|ii|Pn|idjo#t id|ii|i|n|i|ijo|iid d t|iid}|i||jp |tjo5t id |ii|t id |iiPqt id |ii|Pn|}||i}|id jox||i}h}|i o!yt!|i"}Wq}q}Xnt#|i |i|i$|i%||||i| } n|ii|t|iid}||ijot id|ii|Pn|d|_| Sq,Wt|dS(Nsiterator is closedis%s time-stamp reduction at %sRis3%s truncated, possibly due to damaged records at %ss ups %s has invalid status, %s, at %siis'%s has invalid transaction header at %ssIt appears that there is invalid data at the end of the file, possibly due to a system crash. %s truncated to recover from bad data at end.Rus2%s redundant transaction length check failed at %s('RRR!R3RBRdR~RtCorruptedDataErrorRtbufRDRJRRRRmR{t IndexErrorR+RRRyRRvR|RzRoRRRRRtelenR=RtRecordIteratorRRR( RR+RRRRdRRoRR((RRQast    #  !       ( R R RRRJR!RRRaRLRnRzRQ(((RRKs    RcBs#tZdZdZddZRS(s4Iterate over the transactions in a FileStorage file.c CsU||_||_||_||_||_||_ ||_ ||_ | |_dS(N(RDRRRtdescR:Rt _extensionRdRBRt_tendRRRt_tpos( RRDRRRRRdRRR((RRs        ic Cs?|i}x#||ijo|i|}|i}|||ijp|i|i jot i dt i |Pn|||_d}|io|ii|i}nT|idjo d}n:|i|i|it\}}|i|i|i}t|i|i|i|||}|Sq Wt|dS(Ns/%s data record exceeds transaction record at %si(RRBRdRRRRRRRRRRRRmR!RRRRzRRt _loadBackTxnReRIRDtgetTxnFromDatatRecordRR?RR+( RR+RRRRdR?RDR((RRQs(  '     !$ (R R RRRQ(((RRs  RcBstZdZdZRS(sAn abstract database record.cCs:||_||_||_||_||_||_dS(N(ReRRDRRRtdata_txnRd(RReRDRRRRd((RRs      (R R RR(((RRs RcBs/tZddZdZdZdZRS(NcCsL||_||_||_||_||_d|_g|_t|_ dS(Ni( RRRdRRRR!R#RIR (RRRdRRR((RRs       cCs*|i|ijp|idjp|iS(s3Return True if UndoSearch has found enough records.i'N(RR!RRdR (R((RR scCsw|i}|dj oZ|idjp|i|o:|i|ijo|ii|n|id7_ndS(sSearch for another record.iN( Rt _readnextRR!RR!RR#R(RR((RR"s  -c Cs|ii|id|it|iidd8_|ii|i|iit}tt |\} }}}}}|djod|_d Sn|djod Snd}}|o|ii|}n|o|ii|}nh} |o*yt|ii|} Wq0q0Xnhdti| i<dt| i<d|<d |<d |<}|i| |S( s&Read the next record from the storage.iRjiR,RtidRMR9R;R:N(RRRvRdR|RzRRRRRDR}RRRRR R!RRuRR=R(t encodestringtrstripRKRPR( RRRRRRRR}RuRDR((RRs2%!    N (R R R!RRR"R(((RRs   (PRR(tcPickleRiRR=R5R8RRMtloggingttypesRtstructRRtgetattrR!RtZODBR)RRtZODB.POSExceptionRRR2Rtpersistent.TimeStampRKtZODB.lock_fileR"t ZODB.utilsRR|RRtZODB.FileStorage.fspackRAtZODB.FileStorage.formattFileStorageFormatterRRR^R]RRR~tZODB.loglevelsRt ZODB.fsIndexRbR=t getLoggerRRt StorageErrorR R R RtRRRRRtConflictResolvingStorageRRcRfRgRARnRuRKtTransactionRecordRt DataRecordRR(;RRRRKRR]R|RbR2RfRR"R~RRRRgR5RR(RRnRcR^RR R RRRRRRARRKR RRRARRRRRRuRRRRRR=RRRMR)R=RiR8R((Rt?s\          7   "C y  1