m %UIc@s[dZdkZdkZdkZdkZdkZdklZddddddd d d d d ddddddddddddddddddgZ d Z d!Z ei d"jod#k lZlZnaei d$jod#klZlZn:ei d%jod#klZlZnd&Zd'Zeaeed(Zeeed)Zd*Zd+efd,YZhZdfd-YZdefd.YZead/Zead0Zea d1Z!ea"d2Z#d3fd4YZ$d5fd6YZ%d7e%fd8YZ&d9e%fd:YZ'd;e%fd<YZ(ye)Wne*j od=Z+n Xd>Z+d?Z,d@Z-ea.dAZ/ea0dBZ1ea2dCZ3ea4dDZ5ea6dEZ7ea8dFdGZ9ea:dHZ;ea<dIZ=dJZ>ea?dKZ@dLZAeBdMeCdNDZDeDiEdOeCdNDdPZFdQZGdRZHhZIdSdTZJdUdVZKdWdXZLdYZMeiNdZjod[ZOd\ZPd]ZQn?ei d$jod^ZRd_ZQd`ZPneMZQdaZPdbZSdcZTgddZUdeZVeWdfjo eVndS(gsOpen an arbitrary URL. See the following document for more info on URLs: "Names and Addresses, URIs, URLs, URNs, URCs", at http://www.w3.org/pub/WWW/Addressing/Overview.html See also the HTTP spec (from which the error codes are derived): "HTTP - Hypertext Transfer Protocol", at http://www.w3.org/pub/WWW/Protocols/ Related standards and specs: - RFC1808: the "relative URL" spec. (authoritative status) - RFC1738 - the "URL standard". (authoritative status) - RFC1630 - the "URI spec". (informational status) The object returned by URLopener().open(file) will differ per protocol. All you know is that is has methods read(), readline(), readlines(), fileno(), close() and info(). The read*(), fileno() and close() methods work like those of open files. The info() method returns a mimetools.Message object which can be used to query various info about the object, if available. (mimetools.Message objects are queried with the getheader() method.) N(surljointurlopent URLopenertFancyURLopenert urlretrievet urlcleanuptquotet quote_plustunquotet unquote_plust urlencodet url2pathnamet pathname2urltsplittagt localhosttthishostt ftperrorstbasejointunwrapt splittypet splithostt splitusert splitpasswdt splitportt splitnportt splitqueryt splitattrt splitvaluetsplitgophertypet getproxiess1.16i tmac(s url2pathnames pathname2urltnttriscoscCs t|S(sOS-specific conversion from a relative URL of the 'file' scheme to a file system path; not recommended for general use.N(Rtpathname(R ((t#/data/zmath/lib/python2.4/urllib.pyR 4scCs t|S(sOS-specific conversion from a file system path to a relative URL of the 'file' scheme; not recommended for general use.N(RR (R ((R!R 9scCsr|dj otd|}n!tpt}|ant}|djo|i|Sn|i||SdS(s.urlopen(url [, data]) -> open file-like objecttproxiesN(R"tNoneRtopenert _urlopenertdatatopenturl(R(R&R"R$((R!RGs    cCs*tp tanti||||S(N(R%RtretrieveR(tfilenamet reporthookR&(R(R*R+R&((R!RUs cCstotindS(N(R%tcleanup(((R!RZstContentTooShortErrorcBstZdZRS(NcCsti||||_dS(N(tIOErrort__init__tselftmessagetcontent(R0R1R2((R!R/`s(t__name__t __module__R/(((R!R-_scBstZdZeZdeZedZdZdZ dZ dZ edZ edZ ed Zeeed Zed Zed Zd ZeedoedZndZdZdZdZedZRS(s,Class to open URLs. This is a class rather than just a subroutine because we may need more than one set of global protocol-specific options. Note -- this is a base class for those who don't want the automatic handling of errors type 302 (relocated) and 401 (authorization needed).sPython-urllib/%scKs|djo t}n||_|id|_|id|_d|ifg|_ g|_ t i |_ d|_t|_dS(Ntkey_filet cert_files User-agent(R"R#RR0tx509tgetR5R6tversiont addheaderst_URLopener__tempfilestostunlinkt_URLopener__unlinkt tempcachetftpcache(R0R"R7((R!R/rs      cCs|idS(N(R0tclose(R0((R!t__del__scCs|idS(N(R0R,(R0((R!RAscCso|ioFx8|iD]-}y|i|Wqtj oqXqW|i2n|io|iindS(N(R0R;tfileR>tOSErrorR?tclear(R0RC((R!R,s     cGs|ii|dS(sdAdd a header to be used by the HTTP interface only e.g. u.addheader('Accept', 'sound/basic')N(R0R:tappendtargs(R0RG((R!t addheaderscCstt|}|ioF||ijo6|i|\}} t|d}t || |Snt |\}} |p d}n||i joA|i |} t | \}}t|\} }| |f} nd} d|} ||_| idd} t|| p2| o|i| ||SqC|i||Sny>|djot|| | Snt|| | |SWn4tij o%}td|ft i!dnXdS( s6Use URLopener().open(file) instead of open(file, 'r').trbRCtopen_t-t_s socket erroriN("RttoBytestfullurlR0R?R*theadersR'tfpt addinfourlRturltypeR(R"tproxyt proxyhostRthosttselectorR#tnamettypetreplacethasattrtopen_unknown_proxyR&t open_unknowntgetattrtsocketterrortmsgR.tsystexc_info(R0RNR&RTRRR*R`RPRVRURSRWR(RO((R!R's8     cCs(t|\}}tdd|fdS(s/Overridable interface to open unknown URL type.s url errorsunknown url typeN(RRNRXR(R.(R0RNR&R(RX((R!R\scCs,t|\}}tdd||fdS(s/Overridable interface to open unknown URL type.s url errorsinvalid proxy for %sN(RRNRXR(R.RS(R0RSRNR&R(RX((R!R[scCstt|}|io||ijo|i|Snt|\} } |djom| p | djoXy<|i | }|i }~tt| d|fSWqtj o } qXn|i||}|i }|ot|d}ndk} t|\}}t|pd\}}t|pd\}}t|pd\}}tii|d}| i|\}}|i i!|ti"|d}||f}|idj o||i||id| |idd|id d t|n|id | | o|id d | n|o|id|nx|iD]} |i| qW|i|dj o|i |n|i!\}}}|i%}|djot'||d|SnG|djo|i(|||||Sn|i(||||||SdS(sUse HTTP protocol.Nthttps %s://%s%ss http errors no host giventPOSTs Content-types!application/x-www-form-urlencodedsContent-lengths%dtGETt AuthorizationsBasic %stHostishttp:(s http errors no host given()thttplibR#t user_passwdt isinstanceR(tstrRRURVRRtrealhostRRRtresttlowert proxy_bypassR.tbase64t encodestringtstriptauthtHTTPthR&t putrequestt putheaderRyR0R:RGt endheaderstsendtgetreplyterrcodeterrmsgROtgetfileRPRQt http_error(R0R(R&RRRRRORRPRGRRVRURRRRR((R!t open_httpsf               c Csd|}t||oft||}|djo||||||} n|||||||} | o| Sqn|i |||||S(sHandle http errors. Derived class can override this, or provide specific handlers named http_error_DDD where DDD is the 3-digit error code.s http_error_%dN(RRWRZR0R]tmethodR&R#R(RPRRORrthttp_error_default( R0R(RPRRROR&RWRRr((R!RFs   cCs/|i}|itd|||fdS(s>Default error handler: close the connection and raise IOError.s http errorN(RPRutvoidRAR.RRRO(R0R(RPRRROR((R!RUs  tsslcCsdk} d}t|toEt|\} } | o"t | \}} t | } n| }n|\} } t | \}}|}d}|idjo d}nNt|\}}|ot |\}}n|od|||f} n| p tdn|o"dk}|i|i} nd} | i| dd|id|i}|dj o>|id | |id d |id d t|n|id| | o|idd| n|o|id|nx|iD]} |i| qW|i |dj o|i!|n|i"\}}}|i&}|djot(||d|SnG|djo|i)|||||Sn|i)||||||SdS(sUse HTTPS protocol.Nthttpss %s://%s%ss https errors no host giveniR5R6R|s Content-types!application/x-www-form-urlencodedsContent-lengths%dR}R~sBasic %sRishttps:(s https errors no host given(*RR#RRR(RRRURVRRRRRRRRR.RRRRtHTTPSR0R5R6RR&RRRyR:RGRRRRRRORRPRQR(R0R(R&RRRRRORRPRGRVRURRRRRR((R!t open_https\sj                cCsdk}t|\}}|p tdnt|}t|\}}t |\}}t|}|o%t|}|i |||}n|i ||}t|td|S(sUse Gopher protocol.Ns gopher errors no host givensgopher:(s gopher errors no host given(t gopherlibRR(RURVR.RRRXRtqueryt send_queryRPt send_selectorRQt noheaders(R0R(RRURPRRVRX((R!t open_gophers     cCsa|d djo?|dd!djo+|dd!idjo|i|Sn|i|SdS(s/Use local file or FTP depending on form of URL.is//it/i s localhost/N(R(RR0topen_ftpRf(R0R(((R!t open_files?cCsdk} dk}dk} ydkl}Wn tj odkl}nXt|\} }t |}yt i|}Wn0tj o$} t| i| i| inX|i}| ii|idt}| i|d}|i|d|pd||f}| pB|} |d djod |} nt"t#|d || Snt$| \} }| odt&i'| t(t)fjoB|} |d djod |} nt"t#|d || Sntd dS(sUse local file.N(sStringIOtusegmtis6Content-Type: %s Content-Length: %d Last-modified: %s s text/plainiRsfile://RIslocal file errorsnot on local host(slocal file errorsnot on local host(*t mimetypest mimetoolst email.Utilstemailt cStringIOtStringIOt ImportErrorRR(RURCR t localnameR<tstattstatsRDteR.terrnotstrerrorR*tst_sizeRttUtilst formatdatetst_mtimetTruetmodifiedt guess_typetmtypetMessageROturlfileRQR'RtportR^t gethostbynameR R(R0R(RRRCRRRtRRRRRURRRRO((R!Rfs<   )*cCsdk} dk}ydkl}Wn tj odkl}nXt|\}}|p t dnt |\}}t |\}}|ot|\}} nd} t|}t|pd}t| pd} ti|}|pdk} | i}n t|}t|\}}t|}|id} | d | d} }| o| d o| d } n| o| d od| d8st__iter__tnext( RPR0RutreadlineRZR+R,R.R/(R0RP((R!R/0s  cCs d|iit||ifS(Ns<%s at %r whose fp = %r>(R0t __class__R3tidRP(R0((R!t__repr__>scCsLd|_d|_d|_d|_|io|iind|_dS(N(R#R0RuR0R+R,RPRA(R0((R!RABs     (R3R4RR/R3RA(((R!R*-s   R'cBs tZdZdZdZRS(s*Class to add a close hook to an open file.cGs&ti||||_||_dS(N(R*R/R0RPt closehookthookargs(R0RPR4R5((R!R/Ms cCsAti||io&|i|id|_d|_ndS(N(R*RAR0R4R5R#(R0((R!RARs    (R3R4RR/RA(((R!R'Js  taddinfocBs tZdZdZdZRS(s.class to add an info() method to an open file.cCsti||||_dS(N(R*R/R0RPRO(R0RPRO((R!R/\scCs|iS(N(R0RO(R0((R!Rg`s(R3R4RR/Rg(((R!R6Ys  RQcBs)tZdZdZdZdZRS(s9class to add info() and geturl() methods to an open file.cCs&ti||||_||_dS(N(R*R/R0RPROR((R0RPROR(((R!R/fs cCs|iS(N(R0RO(R0((R!RgkscCs|iS(N(R0R((R0((R!tgeturlns(R3R4RR/RgR7(((R!RQcs   cCsdS(Ni((tx((R!t _is_unicodescCs t|tS(N(RR8tunicode(R8((R!R9scCsXt|oGy|id}WqTtj o!tdt|dqTXn|S(stoBytes(u"URL") --> 'URL'.tASCIIsURL s contains non-ASCII charactersN(R9R(tencodet UnicodeErrortrepr(R(((R!RMs #cCsn|i}|d djo(|ddjo|dd!i}n|d djo|di}n|S(s8unwrap('') --> 'type://host/path'.itisURL:N(R(R(R(((R!Rs "cCsztdjodk}|idanti|}|o1|id}|i|t |dfSnd|fS(s:splittype('type:opaquestring') --> 'type', 'opaquestring'.Ns ^([^/:]+):i( t _typeprogR#RtcompileRR(tgroupRRRy(R(RRR((R!Rs  "cCs]tdjodk}|idanti|}|o|iddSnd|fS(s;splithost('//host[:port]/path') --> 'host[:port]', '/path'.Ns^//([^/]*)(.*)$ii(t _hostprogR#RRBRR(RC(R(RR((R!Rs  cCsftdjodk}|idanti|}|ott|iddSnd|fS(sJsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.Ns ^(.*)@(.*)$ii( t _userprogR#RRBRRUtmapRRC(RURR((R!Rs  cCs]tdjodk}|idanti|}|o|iddSn|dfS(s/splitpasswd('user:passwd') -> 'user', 'passwd'.Ns^([^:]*):(.*)$ii(t _passwdprogR#RRBRRRC(RRR((R!Rs  cCs]tdjodk}|idanti|}|o|iddSn|dfS(s*splitport('host:port') --> 'host', 'port'.Ns^(.*):([0-9]+)$ii(t _portprogR#RRBRRURC(RURR((R!Rs  icCstdjodk}|idanti|}|of|idd\}}y$|p tdnt |}Wntj o d}nX||fSn||fS(sSplit host and port, returning numeric port. Return given default port if no ':' found; defaults to -1. Return numerical port if a valid number are found after ':'. Return None if ':' but not a valid number.Ns ^(.*):(.*)$iis no digits( t _nportprogR#RRBRRURCRRRwtnporttdefport(RURKRJRRR((R!Rs     cCs]tdjodk}|idanti|}|o|iddSn|dfS(s/splitquery('/path?query') --> '/path', 'query'.Ns^(.*)\?([^?]*)$ii(t _queryprogR#RRBRR(RC(R(RR((R!Rs  cCs]tdjodk}|idanti|}|o|iddSn|dfS(s)splittag('/path#tag') --> '/path', 'tag'.Ns^(.*)#([^#]*)$ii(t_tagprogR#RRBRR(RC(R(RR((R!R s  cCs!|id}|d|dfS(sksplitattr('/path;attr1=value1;attr2=value2;...') -> '/path', ['attr1=value1', 'attr2=value2', ...].RiiN(R(Rtwords(R(RN((R!R scCs]tdjodk}|idanti|}|o|iddSn|dfS(s-splitvalue('attr=value') --> 'attr', 'value'.Ns^([^=]*)=(.*)$ii(t _valueprogR#RRBRRRC(RRR((R!Rs  cCs?|d djo$|dd!o|d|dfSnd|fS(s2splitgophertype('/Xselector') --> 'X', 'selector'.iRiN(RVR#(RV((R!Rsccs(x!|]}d|t|fVqWdS(s%02xN(t[outmost-iterable]Rtchr(RPR((R!t$siccs(x!|]}d|t|fVqWdS(s%02XN(RPRRQ(RPR((R!RR%scCs|id}xtdt|D]}||}yt|d |d|| 'abc def'.t%iiiRdN( tsRtrestxrangeRyRtitemt _hextochrtKeyErrortUnicodeDecodeErrortunichrRwR(RTRRURW((R!R's .cCs|idd}t|S(s%unquote('%7e/abc+def') -> '~/abc def't+t N(RTRYR(RT((R!R4ssAABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_.-RcCs|tf}yt|}Wnrtj of|t7}h}xBtdD]4}t|}||jo|pd||| 'abc%20def' Each part of a URL, e.g. the path info, the query, etc., has a different set of reserved characters that must be quoted. RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists the following reserved characters. reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | "," Each of these characters is reserved in some component of a URL, but not necessarily in all of them. By default, the quote function is intended for quoting the path section of a URL. Thus, it will not encode '/'. This character is reserved, but in typical usage the quote function is being called on a path where the existing slash characters are used as reserved characters. is%%%02XRdN(Rt always_safetcachekeyt _safemapstsafe_mapRYtrangeRRQtcRFt __getitem__RTRUR(RTRRcR_RaRUR((R!R>s    &RdcCsAd|jo't||d}|iddSnt||S(s9Quote the query fragment of a URL; replacing ' ' with '+'R]R\N(RTRRRY(RTR((R!R`s  ic Cst|do|i}nhy0t|ot|dt o tnWn4tj o(ti\}}} td| nXg}|pTxl|D]E\}}tt|}tt|}|i|d|qWnx|D]\}}tt|}t|to%t|}|i|d|qt|o1t|idd}|i|d|qyt|}Wn:tj o.tt|}|i|d|qXx/|D]'}|i|dtt|qWqWdi|S( svEncode a sequence of two-element tuples or dictionary into a URL query string. If any values in the query arg are sequences and doseq is true, each sequence element is converted to a separate parameter. If the query arg is a sequence of two-element tuples, the order of the parameters in the output will match the order of parameters in the input. titemsis1not a valid non-string sequence or mapping objectRR;RYt&N(RZRReRyRttuplet TypeErrorRaRbttytvattbtltdoseqRRRRRFR9R<R8teltR( RRmRjR8RRiRlRnRRk((R!R gsD "    )cCs`h}xStiiD]B\}}|i}|o#|ddjo|||d proxy server URL mappings. Scan the environment for variables named _proxy; this seems to be the standard convention. If you need a different way, you can pass a proxies dictionary to the [Fancy]URLopener constructor. it_proxyN(R"R<tenvironReRWRR(R"RWR((R!tgetproxies_environments  tdarwincCsy dk}Wntj o hSnXy|i}Wn|ij o hSnXh}d|joD|do9y|d}Wn|ij oqXd||d proxy server URL mappings. By convention the mac uses Internet Config to store proxies. An HTTP proxy, for instance, is stored under the HttpProxy key. Nt UseHTTPProxyt HTTPProxyHosts http://%sR{(ticRtICtconfigR_R"R(RuR"RwR((R!tgetproxies_internetconfigs"   cCsdS(Ni((R8((R!RscCstptS(N(RqRx(((R!Rsc Cshh}y dk}Wntj o |SnXy|i|id}|i|dd}|ot|i|dd}d|jopx|i dD]X}|i dd\}}dk}|id |pd ||f}n||| proxy server URL mappings. Win32 uses the registry to store proxies. Ns;Software\Microsoft\Windows\CurrentVersion\Internet Settingst ProxyEnableit ProxyServerRRis ^([^/:]+)://s%s://%sishttp:R{s http://%ssftp://%sR(R"t_winregRtOpenKeytHKEY_CURRENT_USERtinternetSettingst QueryValueExt proxyEnableRt proxyServerRtptprotocoltaddressRRtCloset WindowsErrorRRh( RRR~RRR{RR"R((R!tgetproxies_registrys<         cCstptS(sReturn a dictionary of scheme -> proxy server URL mappings. Returns settings gathered from the environment, if specified, or the registry. N(RqR(((R!R sc Csydk}dk}Wntj o dSnXyK|i|id}|i|dd} t|i|dd}Wnt j o dSnX| p| odSn|g}y5t i |d}||jo|i|nWnt ij onX|id}d}xi|t|joU||djo6ddt it i t ig|||d +n|d 7}qWxv|D]n}|id d }|id d }|idd }x/|D]'}|i|||iod SqqWqWdS(Nis;Software\Microsoft\Windows\CurrentVersion\Internet SettingsRyt ProxyOverrideRsR s 127.0.0.1it.s\.t*s.*t?(R{RRR|R}R~RRRt proxyOverrideRRUR^RtaddrRFR_RRRyRttestRYtvalRR( RURRRRR~RR{RR((R!RsP           6cCsdS(Ni((RU((R!RKscCsd}x$tdD]}|t|}qW|d}ti}t|}t|}ti}||jo dGHnt |GHt |GHt |GHt ||dGdGHdS(NRdiisWrong!itsec( RTRbRRQRtt0RtqsRtuqstt1R>tround(RRRRRTR((R!ttest1Os           cCsd|||fGHdS(Ns0Block number: %d, Block size: %d, Total size: %d(Rvt blocksizet totalsize(RvRR((R!R+_scCsA|p:dddddg}ttdo|idqAnzx|D]}dd G|Gdd GHt|dt\}}|GH|o6d GHx%|i D]}|d G||GHqWd GHnt |d }|i}~d |jo(tidd}|i|d }n|GHd\}}qKWddGHWdtXdS(Ns /etc/passwdsfile:/etc/passwdsfile://localhost/etc/passwds&ftp://ftp.python.org/pub/python/READMEs http://www.python.org/index.htmlRs!https://synergy.as.cmu.edu/~geek/RKi s======RRIs Rdi((NN(RGRZRRFR(RR#R+tfnRRRR'RPRuR&tstringt maketransttablet translateR(RGRPR(RRR&RR((R!Res6     c Csdk}dk}y#|i|idd\}}Wn&|ij o}|GHdGHdSnXd}xR|D]J\}}|djo|d}n|djodGHdGd GHdSqkqkW|o&|djo t nt |n2|p dGHnx|D]}t |iGqWdS( NitthsUse -h for helpis-ts-hs&Usage: python urllib.py [-t] [url ...]s-t runs self-test;s'otherwise, contents of urls are printed(tgetoptRatargvtoptsRGR_R`tttoRRRR(RRu( RR(RGRRaRR`RR((R!tmains4#        t__main__(XRRR^R<RRaturlparseturljoinRt__all__RRRWt macurl2pathR R t nturl2patht rourl2pathR#R%RRRR.R-R@RRRR RRRRRRRR*R'R6RQR:t NameErrorR9RMRRARRDRRERRGRRHRRIRRLRRMR RRORRtdictRbRXtupdateRRR^R`RRR RqtplatformRxRRRRR+RRR3(:RRRR RR*RR@R6R-RQRRxRRRRRRRR RRMRRR RR9RRR'RRR`RRXRaRRRRR RR^RRqRRR^R+RR RRRRR<R((R!Rs      ]       I              "  @     - 6    !