m %UIc@s[dZdkZdkZdkZdkZdkZdklZddddddd d d d d ddddddddddddddddddgZ d Z d!Z ei d"jod#k lZlZnaei d$jod#klZlZn:ei d%jod#klZlZnd&Zd'Zeaeed(Zeeed)Zd*Zd+efd,YZhZdfd-YZdefd.YZead/Zead0Zea d1Z!ea"d2Z#d3fd4YZ$d5fd6YZ%d7e%fd8YZ&d9e%fd:YZ'd;e%fd<YZ(ye)Wne*j od=Z+n Xd>Z+d?Z,d@Z-ea.dAZ/ea0dBZ1ea2dCZ3ea4dDZ5ea6dEZ7ea8dFdGZ9ea:dHZ;ea<dIZ=dJZ>ea?dKZ@dLZAeBdMeCdNDZDeDiEdOeCdNDdPZFdQZGdRZHhZIdSdTZJdUdVZKdWdXZLdYZMeiNdZjod[ZOd\ZPd]ZQn?ei d$jod^ZRd_ZQd`ZPneMZQdaZPdbZSdcZTgddZUdeZVeWdfjo eVndS(gsOpen an arbitrary URL. See the following document for more info on URLs: "Names and Addresses, URIs, URLs, URNs, URCs", at http://www.w3.org/pub/WWW/Addressing/Overview.html See also the HTTP spec (from which the error codes are derived): "HTTP - Hypertext Transfer Protocol", at http://www.w3.org/pub/WWW/Protocols/ Related standards and specs: - RFC1808: the "relative URL" spec. (authoritative status) - RFC1738 - the "URL standard". (authoritative status) - RFC1630 - the "URI spec". (informational status) The object returned by URLopener().open(file) will differ per protocol. All you know is that is has methods read(), readline(), readlines(), fileno(), close() and info(). The read*(), fileno() and close() methods work like those of open files. The info() method returns a mimetools.Message object which can be used to query various info about the object, if available. (mimetools.Message objects are queried with the getheader() method.) N(surljointurlopent URLopenertFancyURLopenert urlretrievet urlcleanuptquotet quote_plustunquotet unquote_plust urlencodet url2pathnamet pathname2urltsplittagt localhosttthishostt ftperrorstbasejointunwrapt splittypet splithostt splitusert splitpasswdt splitportt splitnportt splitqueryt splitattrt splitvaluetsplitgophertypet getproxiess1.16i tmac(s url2pathnames pathname2urltnttriscoscCs t|S(sOS-specific conversion from a relative URL of the 'file' scheme to a file system path; not recommended for general use.N(Rtpathname(R ((t#/data/zmath/lib/python2.4/urllib.pyR 4scCs t|S(sOS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.N(RR (R ((R!R 9scCsr|dj otd|}n!tpt}|ant}|djo|i|Sn|i||SdS(s.urlopen(url [, data]) -> open file-like objecttproxiesN(R"tNoneRtopenert _urlopenertdatatopenturl(R(R&R"R$((R!RGs    cCs*tp tanti||||S(N(R%RtretrieveR(tfilenamet reporthookR&(R(R*R+R&((R!RUs cCstotindS(N(R%tcleanup(((R!RZstContentTooShortErrorcBstZdZRS(NcCsti||||_dS(N(tIOErrort__init__tselftmessagetcontent(R0R1R2((R!R/`s(t__name__t __module__R/(((R!R-_scBstZdZeZdeZedZdZdZ dZ dZ edZ edZ ed Zeeed Zed Zed Zd ZeedoedZndZdZdZdZedZRS(s,Class to open URLs. This is a class rather than just a subroutine because we may need more than one set of global protocol-specific options. Note -- this is a base class for those who don't want the automatic handling of errors type 302 (relocated) and 401 (authorization needed).sPython-urllib/%scKs|djo t}nt|dp td||_|id|_|id|_ d|i fg|_ g|_ t i|_d|_t|_dS(Nthas_keysproxies must be a mappingtkey_filet cert_files User-agent(R"R#RthasattrtAssertionErrorR0tx509tgetR6R7tversiont addheaderst_URLopener__tempfilestostunlinkt_URLopener__unlinkt tempcachetftpcache(R0R"R:((R!R/rs      cCs|idS(N(R0tclose(R0((R!t__del__scCs|idS(N(R0R,(R0((R!RDscCso|ioFx8|iD]-}y|i|Wqtj oqXqW|i2n|io|iindS(N(R0R>tfileRAtOSErrorRBtclear(R0RF((R!R,s     cGs|ii|dS(sdAdd a header to be used by the HTTP interface only e.g. u.addheader('Accept', 'sound/basic')N(R0R=tappendtargs(R0RJ((R!t addheaderscCstt|}|ioF||ijo6|i|\}} t|d}t || |Snt |\}} |p d}n||i joA|i |} t | \}}t|\} }| |f} nd} d|} ||_| idd} t|| p2| o|i| ||SqC|i||Sny>|djot|| | Snt|| | |SWn4tij o%}td|ft i!dnXdS( s6Use URLopener().open(file) instead of open(file, 'r').trbRFtopen_t-t_s socket erroriN("RttoBytestfullurlR0RBR*theadersR'tfpt addinfourlRturltypeR(R"tproxyt proxyhostRthosttselectorR#tnamettypetreplaceR8topen_unknown_proxyR&t open_unknowntgetattrtsocketterrortmsgR.tsystexc_info(R0RQR&RWRUR*RbRSRYRXRVRZR(RR((R!R's8     cCs(t|\}}tdd|fdS(s/Overridable interface to open unknown URL type.s url errorsunknown url typeN(RRQR[R(R.(R0RQR&R(R[((R!R^scCs,t|\}}tdd||fdS(s/Overridable interface to open unknown URL type.s url errorsinvalid proxy for %sN(RRQR[R(R.RV(R0RVRQR&R(R[((R!R]scCstt|}|io||ijo|i|Snt|\} } |djom| p | djoXy<|i | }|i }~tt| d|fSWqtj o } qXn|i||}|i }|ot|d}ndk} t|\}}t|pd\}}t|pd\}}t|pd\}}tii|d}| i|\}}|i i!|ti"|d}||f}|idj o||i|RItfdopentresulttbstsizetreadtblocknumR+tinttblocktlentwriteRDR-(R0R(R*R+R&RpRmRtRxRvRlRgRbR[RSRjRwRrRuRnRkRRRz((R!R)sn"             cCsdk} d}t|toEt|\} } | o"t | \}} t | } n| }n|\} } t | \}}|}d}|idjo d}net|\}}|ot |\}}n|od|||f} nt|o |} n| p tdn|o"dk}|i|i} nd} | i| }|dj o>|id| |idd|id d t|n|id | | o|id d | n|o|id|nx|iD]} |i| qW|i|dj o|i |n|i!\}}}|i%}|djot'||d|SnG|djo|i(|||||Sn|i(||||||SdS(sUse HTTP protocol.Nthttps %s://%s%ss http errors no host giventPOSTs Content-types!application/x-www-form-urlencodedsContent-lengths%dtGETt AuthorizationsBasic %stHostishttp:(s http errors no host given()thttplibR#t user_passwdt isinstanceR(tstrRRXRYRRtrealhostRRUtresttlowert proxy_bypassR.tbase64t encodestringtstriptauthtHTTPthR&t putrequestt putheaderR{R0R=RJt endheaderstsendtgetreplyterrcodeterrmsgRRtgetfileRSRTt http_error(R0R(R&RRRURRRRSRJRRYRXRRRRR((R!t open_httpsf               c Csd|}t||oft||}|djo||||||} n|||||||} | o| Sqn|i |||||S(sHandle http errors. Derived class can override this, or provide specific handlers named http_error_DDD where DDD is the 3-digit error code.s http_error_%dN(RRZR8R0R_tmethodR&R#R(RSRRRRtthttp_error_default( R0R(RSRRRRR&RZRRt((R!RFs   cCs/|i}|itd|||fdS(s>Default error handler: close the connection and raise IOError.s http errorN(RSRwtvoidRDR.RRRR(R0R(RSRRRRR((R!RUs  tsslcCsdk} d}t|toEt|\} } | o"t | \}} t | } n| }n|\} } t | \}}|}d}|idjo d}nNt|\}}|ot |\}}n|od|||f} n| p tdn|o"dk}|i|i} nd} | i| dd|id|i}|dj o>|id | |id d |id d t|n|id| | o|idd| n|o|id|nx|iD]} |i| qW|i |dj o|i!|n|i"\}}}|i&}|djot(||d|SnG|djo|i)|||||Sn|i)||||||SdS(sUse HTTPS protocol.Nthttpss %s://%s%ss https errors no host giveniR6R7R~s Content-types!application/x-www-form-urlencodedsContent-lengths%dRRsBasic %sRishttps:(s https errors no host given(*RR#RRR(RRRXRYRRRRRURRR.RRRRtHTTPSR0R6R7RR&RRR{R=RJRRRRRRRRRSRTR(R0R(R&RRRURRRRSRJRYRXRRRRRR((R!t open_https\sj                cCsdk}t|\}}|p tdnt|}t|\}}t |\}}t|}|o%t|}|i |||}n|i ||}t|td|S(sUse Gopher protocol.Ns gopher errors no host givensgopher:(s gopher errors no host given(t gopherlibRR(RXRYR.RRR[Rtqueryt send_queryRSt send_selectorRTt noheaders(R0R(RRXRSRRYR[((R!t open_gophers     cCsa|d djo?|dd!djo+|dd!idjo|i|Sn|i|SdS(s/Use local file or FTP depending on form of URL.is//it/i s localhost/N(R(RR0topen_ftpRh(R0R(((R!t open_files?cCsdk} dk}dk} ydkl}Wn tj odkl}nXt|\} }t |}yt i|}Wn0tj o$} t| i| i| inX|i}| ii|idt}| i|d}|i|d|pd||f}| pB|} |d djod |} nt"t#|d || Snt$| \} }| odt&i'| t(t)fjoB|} |d djod |} nt"t#|d || Sntd dS(sUse local file.N(sStringIOtusegmtis6Content-Type: %s Content-Length: %d Last-modified: %s s text/plainiRsfile://RLslocal file errorsnot on local host(slocal file errorsnot on local host(*t mimetypest mimetoolst email.Utilstemailt cStringIOtStringIOt ImportErrorRR(RXRFR t localnameR?tstattstatsRGteR.terrnotstrerrorR*tst_sizeRvtUtilst formatdatetst_mtimetTruetmodifiedt guess_typetmtypetMessageRRturlfileRTR'RtportR`t gethostbynameR R(R0R(RRRFRRRvRRRRRXRRRRR((R!Rhs<   )*cCsdk} dk}ydkl}Wn tj odkl}nXt|\}}|p t dnt |\}}t |\}}|ot|\}} nd} t|}t|pd}t| pd} ti|}|pdk} | i}n t|}t|\}}t|}|id} | d | d} }| o| d o| d } n| o| d od| dt __version__R<R/RERDR,RKR'R^R]R)RRRR8R`RRRRhRR(((R!Res,        !  ? 8  9   # 8st__iter__tnext( RSR0RwtreadlineR8R-R.R0R1(R0RS((R!R/0s  cCs d|iit||ifS(Ns<%s at %r whose fp = %r>(R0t __class__R3tidRS(R0((R!t__repr__>scCsLd|_d|_d|_d|_|io|iind|_dS(N(R#R0RwR2R-R.RSRD(R0((R!RDBs     (R3R4RR/R5RD(((R!R,-s   R)cBs tZdZdZdZRS(s*Class to add a close hook to an open file.cGs&ti||||_||_dS(N(R,R/R0RSt closehookthookargs(R0RSR6R7((R!R/Ms cCsAti||io&|i|id|_d|_ndS(N(R,RDR0R6R7R#(R0((R!RDRs    (R3R4RR/RD(((R!R)Js  taddinfocBs tZdZdZdZRS(s.class to add an info() method to an open file.cCsti||||_dS(N(R,R/R0RSRR(R0RSRR((R!R/\scCs|iS(N(R0RR(R0((R!Ri`s(R3R4RR/Ri(((R!R8Ys  RTcBs)tZdZdZdZdZRS(s9class to add info() and geturl() methods to an open file.cCs&ti||||_||_dS(N(R,R/R0RSRRR((R0RSRRR(((R!R/fs cCs|iS(N(R0RR(R0((R!RikscCs|iS(N(R0R((R0((R!tgeturlns(R3R4RR/RiR9(((R!RTcs   cCsdS(Ni((tx((R!t _is_unicodescCs t|tS(N(RR:tunicode(R:((R!R;scCsXt|oGy|id}WqTtj o!tdt|dqTXn|S(stoBytes(u"URL") --> 'URL'.tASCIIsURL s contains non-ASCII charactersN(R;R(tencodet UnicodeErrortrepr(R(((R!RPs #cCsn|i}|d djo(|ddjo|dd!i}n|d djo|di}n|S(s8unwrap('') --> 'type://host/path'.itisURL:N(R(R(R(((R!Rs "cCsztdjodk}|idanti|}|o1|id}|i|t |dfSnd|fS(s:splittype('type:opaquestring') --> 'type', 'opaquestring'.Ns ^([^/:]+):i( t _typeprogR#RtcompileRR(tgroupRRR{(R(RRR((R!Rs  "cCs]tdjodk}|idanti|}|o|iddSnd|fS(s;splithost('//host[:port]/path') --> 'host[:port]', '/path'.Ns^//([^/]*)(.*)$ii(t _hostprogR#RRDRR(RE(R(RR((R!Rs  cCsftdjodk}|idanti|}|ott|iddSnd|fS(sJsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.Ns ^(.*)@(.*)$ii( t _userprogR#RRDRRXtmapRRE(RXRR((R!Rs  cCs]tdjodk}|idanti|}|o|iddSn|dfS(s/splitpasswd('user:passwd') -> 'user', 'passwd'.Ns^([^:]*):(.*)$ii(t _passwdprogR#RRDRRRE(RRR((R!Rs  cCs]tdjodk}|idanti|}|o|iddSn|dfS(s*splitport('host:port') --> 'host', 'port'.Ns^(.*):([0-9]+)$ii(t _portprogR#RRDRRXRE(RXRR((R!Rs  icCstdjodk}|idanti|}|of|idd\}}y$|p tdnt |}Wntj o d}nX||fSn||fS(sSplit host and port, returning numeric port. Return given default port if no ':' found; defaults to -1. Return numerical port if a valid number are found after ':'. Return None if ':' but not a valid number.Ns ^(.*):(.*)$iis no digits( t _nportprogR#RRDRRXRERRRytnporttdefport(RXRMRLRRR((R!Rs     cCs]tdjodk}|idanti|}|o|iddSn|dfS(s/splitquery('/path?query') --> '/path', 'query'.Ns^(.*)\?([^?]*)$ii(t _queryprogR#RRDRR(RE(R(RR((R!Rs  cCs]tdjodk}|idanti|}|o|iddSn|dfS(s)splittag('/path#tag') --> '/path', 'tag'.Ns^(.*)#([^#]*)$ii(t_tagprogR#RRDRR(RE(R(RR((R!R s  cCs!|id}|d|dfS(sksplitattr('/path;attr1=value1;attr2=value2;...') -> '/path', ['attr1=value1', 'attr2=value2', ...].RiiN(R(Rtwords(R(RP((R!R scCs]tdjodk}|idanti|}|o|iddSn|dfS(s-splitvalue('attr=value') --> 'attr', 'value'.Ns^([^=]*)=(.*)$ii(t _valueprogR#RRDRRRE(RRR((R!Rs  cCs?|d djo$|dd!o|d|dfSnd|fS(s2splitgophertype('/Xselector') --> 'X', 'selector'.iRiN(RYR#(RY((R!Rsccs(x!|]}d|t|fVqWdS(s%02xN(t[outmost-iterable]Rtchr(RRR((R!t$siccs(x!|]}d|t|fVqWdS(s%02XN(RRRRS(RRR((R!RT%scCs|id}xtdt|D]}||}yt|d |d|| 'abc def'.t%iiiRfN( tsRtrestxrangeR{Rtitemt _hextochrtKeyErrortUnicodeDecodeErrortunichrRyR(RVRRWRY((R!R's .cCs|idd}t|S(s%unquote('%7e/abc+def') -> '~/abc def't+t N(RVR\R(RV((R!R4ssAABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-RcCs|tf}yt|}Wnrtj of|t7}h}xBtdD]4}t|}||jo|pd||| 'abc%20def' Each part of a URL, e.g. the path info, the query, etc., has a different set of reserved characters that must be quoted. RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists the following reserved characters. reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | "," Each of these characters is reserved in some component of a URL, but not necessarily in all of them. By default, the quote function is intended for quoting the path section of a URL. Thus, it will not encode '/'. This character is reserved, but in typical usage the quote function is being called on a path where the existing slash characters are used as reserved characters. is%%%02XRfN(Rt always_safetcachekeyt _safemapstsafe_mapR[trangeRRStcRHt __getitem__RVRWR(RVRReRaRcRWR((R!R>s    &RfcCsAd|jo't||d}|iddSnt||S(s9Quote the query fragment of a URL; replacing ' ' with '+'R_R^N(RVRRR\(RVR((R!R`s  ic Cst|do|i}nhy0t|ot|dt o tnWn4tj o(ti\}}} td| nXg}|pTxl|D]E\}}tt|}tt|}|i|d|qWnx|D]\}}tt|}t|to%t|}|i|d|qt|o1t|idd}|i|d|qyt|}Wn:tj o.tt|}|i|d|qXx/|D]'}|i|dtt|qWqWdi|S( svEncode a sequence of two-element tuples or dictionary into a URL query string. If any values in the query arg are sequences and doseq is true, each sequence element is converted to a separate parameter. If the query arg is a sequence of two-element tuples, the order of the parameters in the output will match the order of parameters in the input. titemsis1not a valid non-string sequence or mapping objectRR=R\t&N(R8RRgR{Rttuplet TypeErrorRcRdttytvattbtltdoseqRRRRRIR;R>R:teltR( RRoRlR:RRkRnRpRRm((R!R gsD "    )cCs`h}xStiiD]B\}}|i}|o#|ddjo|||d proxy server URL mappings. Scan the environment for variables named _proxy; this seems to be the standard convention. If you need a different way, you can pass a proxies dictionary to the [Fancy]URLopener constructor. it_proxyN(R"R?tenvironRgRZRR(R"RZR((R!tgetproxies_environments  tdarwincCsy dk}Wntj o hSnXy|i}Wn|ij o hSnXh}d|joD|do9y|d}Wn|ij oqXd||d proxy server URL mappings. By convention the mac uses Internet Config to store proxies. An HTTP proxy, for instance, is stored under the HttpProxy key. Nt UseHTTPProxyt HTTPProxyHosts http://%sR}(ticRtICtconfigRaR"R(RwR"RyR((R!tgetproxies_internetconfigs"   cCsdS(Ni((R:((R!RscCstptS(N(RsRz(((R!Rsc Cshh}y dk}Wntj o |SnXy|i|id}|i|dd}|ot|i|dd}d|jopx|i dD]X}|i dd\}}dk}|id |pd ||f}n||| proxy server URL mappings. Win32 uses the registry to store proxies. Ns;Software\Microsoft\Windows\CurrentVersion\Internet Settingst ProxyEnableit ProxyServerRRis ^([^/:]+)://s%s://%sishttp:R}s http://%ssftp://%sR(R"t_winregRtOpenKeytHKEY_CURRENT_USERtinternetSettingst QueryValueExt proxyEnableRt proxyServerRtptprotocoltaddressRRtCloset WindowsErrorRRj( RRRRRR}RR"R((R!tgetproxies_registrys<         cCstptS(sReturn a dictionary of scheme -> proxy server URL mappings. Returns settings gathered from the environment, if specified, or the registry. N(RsR(((R!R sc Csydk}dk}Wntj o dSnXyK|i|id}|i|dd} t|i|dd}Wnt j o dSnX| p| odSn|g}y5t i |d}||jo|i|nWnt ij onX|id}d}xi|t|joU||djo6ddt it i t ig|||d +n|d 7}qWxv|D]n}|id d }|id d }|idd }x/|D]'}|i|||iod SqqWqWdS(Nis;Software\Microsoft\Windows\CurrentVersion\Internet SettingsR{t ProxyOverrideRsR s 127.0.0.1it.s\.t*s.*t?(R}RRR~RRRRRt proxyOverrideRRXR`RtaddrRIRaRRR{RttestR\tvalRR( RXRRRRRRR}RR((R!RsP           6cCsdS(Ni((RX((R!RKscCsd}x$tdD]}|t|}qW|d}ti}t|}t|}ti}||jo dGHnt |GHt |GHt |GHt ||dGdGHdS(NRfiisWrong!itsec( RVRdRRSRtt0RtqsRtuqstt1R@tround(RRRRRVR((R!ttest1Os           cCsd|||fGHdS(Ns0Block number: %d, Block size: %d, Total size: %d(Rxt blocksizet totalsize(RxRR((R!R+_scCsA|p:dddddg}ttdo|idqAnzx|D]}dd G|Gdd GHt|dt\}}|GH|o6d GHx%|i D]}|d G||GHqWd GHnt |d }|i}~d |jo(tidd}|i|d }n|GHd\}}qKWddGHWdtXdS(Ns /etc/passwdsfile:/etc/passwdsfile://localhost/etc/passwds&ftp://ftp.python.org/pub/python/READMEs http://www.python.org/index.htmlRs!https://synergy.as.cmu.edu/~geek/RNi s======RRLs Rfi((NN(RJR8RRIR(RR#R+tfnRRRR'RSRwR&tstringt maketransttablet translateR(RJRSR(RRR&RR((R!Res6     c Csdk}dk}y#|i|idd\}}Wn&|ij o}|GHdGHdSnXd}xR|D]J\}}|djo|d}n|djodGHdGd GHdSqkqkW|o&|djo t nt |n2|p dGHnx|D]}t |iGqWdS( NitthsUse -h for helpis-ts-hs&Usage: python urllib.py [-t] [url ...]s-t runs self-test;s'otherwise, contents of urls are printed(tgetoptRctargvtoptsRJRaRbtttoRRRR(RRw( RR(RJRRcRRbRR((R!tmains4#        t__main__(XRRR`R?RRcturlparseturljoinRt__all__RRRZt macurl2pathR R t nturl2patht rourl2pathR#R%RRRR.R-RCRRRR RRRRRRRR,R)R8RTR<t NameErrorR;RPRRCRRFRRGRRIRRJRRKRRNRROR RRQRRtdictRdRZtupdateRRR`RbRRR RstplatformRzRRRRR+RRR3(:RRRR RR,RRCR8R-RTRRzRRRRRRRR RRPRRR RR;RRR)RRRbRRZRcRRRRR RR`RRsRRR`R+RR RRRRR?R((R!Rs      ]       I              "  @     - 6    !