ZQdsk@dZdZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z m Z mZddlmZddlmZmZdd lmZdd ZGd d eZdZdZdZdZdZGddeZGddeZ GddeZ!Gdde Z"dS)zRefactoring framework. Used as a main program, this can refactor any number of files and/or recursively descend down directories. Imported as a module, this provides infrastructure to write your own refactoring tool. z#Guido van Rossum N)chain)drivertokenizetoken) find_root)pytreepygram) btm_matcherTct|ggdg}g}tj|jD]<\}}}|dr!|r |dd}||=|S)zEReturn a sorted list of all available fix names in the given package.*fix_N) __import__pkgutil iter_modules__path__ startswithappend) fixer_pkg remove_prefixpkg fix_namesfindernameispkgs '/usr/lib/python3.11/lib2to3/refactor.pyget_all_fix_namesrs YB . .CI&3CLAA##e ??6 " " # ABBx   T " " " ceZdZdS) _EveryNodeN__name__ __module__ __qualname__rrr!r!+Drr!ct|tjtjfr|jt |jhSt|tjr"|jrt|jSt t|tj rAt}|jD])}|D]$}| t|%*|Std|z)zf Accepts a pytree Pattern Node and returns a set of the pattern types which will match first. Nz$Oh no! I don't understand pattern %s) isinstancer NodePattern LeafPatterntyper!NegatedPatterncontent_get_head_typesWildcardPatternsetupdate Exception)patrpxs rr/r//s#*F,>?@@ 8  z#v,-- ; 0"3;// /#v-.. EE - -A - -++,,,, - :SA B BBrcZtjt}g}|D]}|jr[ t |j}|D]}|||?#t $r||Y`wxYw|j!||j|||ttj j tj j D]}|||t|S)z^ Accepts a list of fixers and returns a dictionary of head node type --> fixer list. ) collections defaultdictlistpatternr/rr! _accept_typerr python_grammar symbol2numbervaluestokensextenddict) fixer_list head_nodeseveryfixerheads node_types r_get_headnode_dictrJKsL(..J E $ $ = $ 8' 66"'88Iy)0077778 $ $ $ U##### $ !-5-.55e<<<< U####60>EEGG!0799,, 9$$U++++   sAA?>A?c<fdtdDS)zN Return the fully qualified names for fixers in the package pkg_name. c g|] }dz|z S.r&).0fix_namepkg_names r z+get_fixers_from_package..hs8 @ @ @ sNX % @ @ @rF)r)rQs`rget_fixers_from_packagerSds@ @ @ @ @-h>> @ @ @@rc|SNr&)objs r _identityrWks Jrchd}tjtj|jfd}t t jtjt j h}t} |\}}||vr|t j kr|rnd}n|t j kr|dkr|\}}|t j ks|dkrn|\}}|t j ks|dkrn|\}}|t j kr|dkr |\}}|t j krV|||\}}|t j ks|dkrn|\}}|t j kVnn n#t$rYnwxYwt |S) NFcBt}|d|dfS)Nrr)next)tokgens radvancez(_detect_future_features..advancers 3ii1vs1v~rTfrom __future__import(,)rgenerate_tokensioStringIOreadline frozensetrNEWLINENLCOMMENTr1STRINGNAMEOPadd StopIteration)sourcehave_docstringr]ignorefeaturestpvaluer\s @r_detect_future_featuresrvosN  "2;v#6#6#? @ @C x{EMB C CFuuH   IBV||u|##!!%uz!!evoo#GII E##u '<'<#GII E##u'8'8#GII E>>esll ' IBEJ&&LL''' ' IBUX~~# ' IB EJ&&3 4      X  s3D!F F"!F"ceZdZdZdS) FixerErrorzA fixer could not be loaded.N)r#r$r%__doc__r&rrrxrxs&&&&rrxceZdZddddZdZdZddZdZdZd Z d Z d Z dd Z dd Z dZddZdZd dZdZdZ d!dZd"dZdZdZdZdZdZdZdZdZdS)#RefactoringToolF)print_function exec_functionwrite_unchanged_filesFixrNcP||_|pg|_|j|_||j|t j|_|jdr|jj d=n|jdr |jj d=|j d|_ g|_ tjd|_g|_d|_t%j|jt(j|j |_|\|_|_g|_t5j|_g|_g|_t?|j|jD]k}|j r|j!|$||jvr|j"|H||jvr|j"|ltG|j|_$tG|j|_%dS) zInitializer. Args: fixer_names: a list of fixers to import options: a dict with configuration. explicit: a list of fixers to run even if they are explicit. Nr|printr}execr~r{F)convertlogger)&fixersexplicit_default_optionscopyoptionsr2r r>grammarkeywordsgetr~errorslogging getLoggerr fixer_logwroterDriverr r get_fixers pre_order post_orderfilesbm BottomMatcherBM bmi_pre_orderbmi_post_orderr BM_compatible add_fixerrrJbmi_pre_order_headsbmi_post_order_heads)self fixer_namesrrrGs r__init__zRefactoringTool.__init__s"  B ,1133   L   ( ( (,1133 <( ) . %g.. \/ * . %f- &*\%5%56M%N%N" '(9::  mDL,2N+/;888 +///*;*;' "$$ 4?DN;; 2 2E" 2!!%(((($.(("))%0000$/))#**5111#5d6H#I#I $6t7J$K$K!!!rcg}g}|jD]}t|iidg}|ddd}||jr|t |jd}|d}|jdd|Dz} t||}n$#t$rtd |d|dwxYw||j |j } | jr*|jd ur!||jvr|d |!|d || jd kr|| Y| jdkr|| {td| jzt'jd} || || ||fS)aInspects the options to load the requested patterns and handlers. Returns: (pre_order, post_order), where pre_order is the list of fixers that want a pre-order AST traversal, and post_order is the list that want post-order traversal. r rNrN_c6g|]}|Sr&)title)rOr6s rrRz.RefactoringTool.get_fixers..s 5O5O5OAaggii5O5O5Orz Can't find TzSkipping optional fixer: %szAdding transformation: %sprepostzIllegal fixer order: %r run_orderkey)rrrsplitr FILE_PREFIXlensplit CLASS_PREFIXjoingetattrAttributeErrorrxrrr log_message log_debugorderroperator attrgettersort) rpre_order_fixerspost_order_fixers fix_mod_pathmodrPparts class_name fix_classrGkey_funcs rrzRefactoringTool.get_fixerss" K J JL\2rC599C#**32226H""4#344 <#C(8$9$9$:$:;NN3''E*RWW5O5O5O5O5O-P-PPJ X#C44 ! X X X jxxx!LMMSWW XIdlDN;;E~ $-t";";  55  !>III NN6 A A A{e## ''....&&!((//// !:U[!HIII&{33(+++8,,, "344s 1C!C#c)zCalled when an error occurs.r&)rmsgargskwdss r log_errorzRefactoringTool.log_errors rcH|r||z}|j|dS)zHook to log a message.N)rinforrrs rrzRefactoringTool.log_messages/  *C rcH|r||z}|j|dSrU)rdebugrs rrzRefactoringTool.log_debug s/  *C #rcdS)zTCalled with the old version, new version, and filename of a refactored file.Nr&)rold_textnew_textfilenameequals r print_outputzRefactoringTool.print_outputs  rc|D]P}tj|r||||9||||QdS)z)Refactor a list of files and directories.N)ospathisdir refactor_dir refactor_file)ritemswrite doctests_only dir_or_files rrefactorzRefactoringTool.refactorsn! F FKw}}[)) F!!+umDDDD"";}EEEE  F Frctjdz}tj|D]\}}}|d||||D]w}|ds`tj|d|kr7tj||} | | ||xd|D|dd<dS)zDescends down a directory and refactor every Python file found. Python files are assumed to have a .py extension. Files and subdirectories starting with '.' are skipped. pyzDescending into %srNrc<g|]}|d|SrM)r)rOdns rrRz0RefactoringTool.refactor_dir..2s)KKK" c8J8JK2KKKrN) rextsepwalkrrrrsplitextrr) rdir_namerrpy_extdirpathdirnames filenamesrfullnames rrzRefactoringTool.refactor_dir sT!,.GH,=,= L L (GXy NN/ 9 9 9 MMOOO NN   ! G G,,GG$$T**1-77!w||GT::H&&x FFFKKKKKHQQQKK L Lrc t|d}n/#t$r"}|d||Yd}~dSd}~wwxYw tj|jd}|n#|wxYwtj|d|d5}||fcdddS#1swxYwYdS) zG Do our best to decode a Python source file correctly. rbzCan't open %s: %sNNNrr5rencodingnewline) openOSErrorrrdetect_encodingrfcloserdread)rrferrrs r_read_python_sourcez#RefactoringTool._read_python_source4s" Xt$$AA    NN.# > > >:::::  / ;;A>H GGIIIIAGGIIII WXsXr B B B &a6688X% & & & & & & & & & & & & & & & & & &s. ?:?A77B (C  CCc||\}}|dS|dz }|rl|d||||}|js||kr||||||dS|d|dS|||}|js |r7|jr0|t|dd|||dS|d|dS)zRefactors a file.N zRefactoring doctests in %szNo doctest changes in %sr)rrzNo changes in %s)rrrefactor_docstringr~processed_filerefactor_string was_changedstr)rrrrinputroutputtrees rrzRefactoringTool.refactor_fileDs@228<<x = F    = NN7 B B B,,UH==F) EVu__##FHeUHMMMMM98DDDDD''x88D) =d =t7G =##CIIcrcNH*/($DDDDD18<<<<}|d||jj |Yd}~|j|j_dSd}~wwxYw |j|j_n#|j|j_wxYw||_ | d|| |||S)aFRefactor a given input string. Args: data: a string holding the code to be refactored. name: a human-readable name for use in error/log messages. Returns: An AST corresponding to the refactored input stream; None if there were errors during the parse. r|zCan't parse %s: %s: %sNzRefactoring %s) rvr !python_grammar_no_print_statementrr parse_stringr3r __class__r#future_featuresr refactor_tree)rdatarrsrrs rrzRefactoringTool.refactor_string[s +400 x ' '"("JDK  /;++D11DD    NN3!7 > > > FFF"&,DK       #',DK  $,DK  . . . .' '... 4&&& s/AB$ B"B 2B$ BB$$B7ctj}|rh|d||d}|js||kr||d|dS|ddS||d}|js |r-|jr&|t|d|dS|ddS)NzRefactoring doctests in stdinzzNo doctest changes in stdinzNo changes in stdin) sysstdinrrrr~rrrr)rrrrrs rrefactor_stdinzRefactoringTool.refactor_stdinvs     6 NN: ; ; ;,,UI>>F) >Vu__##FIu=====<=====''y99D) 6d 6t7G 6##CIIy%@@@@@455555rct|j|jD]}|||||j|||j||j| }t| r|jj D]}||vr||r|| tjjd|jr+|| tjjt'||D]8}|||vr||| t+|n#t,$rYEwxYw|jr ||jvrZ||}|r|||}||||D]*}|jsg|_|j|+|j| }|D],} | |vrg|| <|| || -:t| t|j|jD]}||||jS)aRefactors a parse tree (modifying the tree in place). For compatible patterns the bottom matcher module is used. Otherwise the tree is traversed node-to-node for matches. Args: tree: a pytree.Node instance representing the root of the tree to be refactored. name: a human-readable name for this tree. Returns: True if the tree was modified, False otherwise. T)rreverser)rrr start_tree traverse_byrrrrunleavesanyr@rrr Basedepthkeep_line_order get_linenor;remover ValueErrorfixers_appliedmatch transformreplacerrB finish_treer) rrrrG match_setnoderesultsnew new_matchesfxrs rr zRefactoringTool.refactor_trees% 4>4?;; ) )E   T4 ( ( ( ( 14>>3C3CDDD 2DOO4E4EFFFGKK .. )""$$%%/ L. L. LI%%)E*:%e$))fk.?)NNN,J"%(--&+2H-III $Yu%5 6 6$L$L9U#333%e,33D999%%dOOOO)%%%%H%  .%5D%A>@(;$($7$>$>u$E$E$E$E/3gkk#**,,.G.G +6!L!LC+.)+;+;79 #$-cN$9$9+c:J$K$K$K$K_)""$$%%/ Lb4>4?;; * *E   dD ) ) ) )sF%% F21F2c|sdS|D]X}||jD]H}||}|r/|||}||||}IYdS)aTraverse an AST, applying a set of fixers to each node. This is a helper method for refactor_tree(). Args: fixers: a list of fixer instances. traversal: a generator that yields AST nodes. Returns: None N)r,rrr)rr traversalr"rGr#r$s rrzRefactoringTool.traverse_bys  F # #D * # #++d++#//$88C S)))"  # # #rc^|j||||d}|dS||k}||||||r|d||jsdS|r|||||dS|d|dS)zR Called when a file has been refactored and there may be changes. NrzNo changes to %szNot writing changes to %s)rrrrrr~ write_file)rrrrrrrs rrzRefactoringTool.processed_files (###  //99!>>   NN-x 8 8 8-   B OOHh( C C C C C NN6 A A A A Arc tj|d|d}n/#t$r"}|d||Yd}~dSd}~wwxYw|5 ||n.#t$r!}|d||Yd}~nd}~wwxYwdddn #1swxYwY|d|d|_dS) zWrites a string to a file. It first shows a unified diff between the old text and the new text, and then rewrites the file; the latter is only done if the write option is set. wrrzCan't create %s: %sNzCan't write %s: %szWrote changes to %sT)rdrrrrrr)rrrrrfprs rr*zRefactoringTool.write_filesX 32FFFBB    NN0(C @ @ @ FFFFF  D D D"""" D D D3XsCCCCCCCC D D D D D D D D D D D D D D D D ,h777 sP AAA BA$#B$ B.B B BBB"%B"z>>> z... c g}d}d}d}d}|dD])}|dz }||jrW|+|||||||}|g}||j} |d| }|V|||jzs#|||jzdzkr| ||+||||||d}d}| |+|+||||||d |S)aRefactors a docstring, looking for doctests. This returns a modified version of the input string. It looks for doctests, which start with a ">>>" prompt, and may be continued with "..." prompts, as long as the "..." is indented the same as the ">>>". (Unfortunately we can't use the doctest module's parser, since, like most parsers, it is not geared towards preserving the original source.) NrTkeependsrrr) splitlineslstriprPS1rBrefactor_doctestfindPS2rstriprr) rrrresultblock block_linenoindentlinenolineis rrz"RefactoringTool.refactor_docstrings $$d$33 $ $D aKF{{}}''11 $$MM$"7"7|8>#J#JKKK% IIdh''bqb$??6DH#455%6DHOO$5$55<<< T""""$MM$"7"7|8>#J#JKKK d####   MM$//|06BB C C Cwwvrc ||}n#t$r}jtjr.|D]+}d|d,d|||j j ||cYd}~Sd}~wwxYw ||rt| d}|d|dz ||dz d}} | dg|dz zks J| |dds|dxxdz cc<jz|d zg}|r|fd |Dz }|S) zRefactors one doctest. A doctest is given as a block of lines, the first of which starts with ">>>" (possibly indented), while the remaining lines start with "..." (identically indented). z Source: %srz+Can't parse docstring in %s line %s: %s: %sNTr/rrrc*g|]}jz|zSr&)r6)rOr=r;rs rrRz4RefactoringTool.refactor_doctest..^s%CCCt&48+d2CCCr) parse_blockr3r isEnabledForrDEBUGrr7rrr#r rr1endswithr3pop) rr9r<r;rrrr=r$clippeds ` ` rr4z RefactoringTool.refactor_doctestDs ##E66::DD   {'' 66 D!DDDNN<T1B1BCCCC NNH#VS]-CS J J JLLLLLL     dH - - Dd))&&&55Cyqy>3vaxyy>SGtfq11117111r7##D)) B4dh&34E DCCCCCsCCCC s B'A6B"B'"B'c6|jrd}nd}|js|d|n5|d||jD]}|||jr4|d|jD]}|||jrut |jdkr|dn(|dt |j|jD]\}}}|j|g|Ri|dSdS) Nwerez need to bezNo files %s modified.zFiles that %s modified:z$Warnings/messages while refactoring:rzThere was 1 error:zThere were %d errors:)rrrrrr)rrHfilemessagerrrs r summarizezRefactoringTool.summarizeask : DDDz '   4d ; ; ; ;   6 = = =  ' '  &&&& > *   C D D D> * *  )))) ; 54;1$$  !56666  !8#dk:J:JKKK#'; 5 5T4  4t444t4444  5 5  5 5rc|j||||}t|_|S)zParses a block into a tree. This is necessary to get correct line number / offset information in the parser diagnostics and embedded into the parse tree. )r parse_tokens wrap_toksrgr)rr9r<r;rs rrAzRefactoringTool.parse_blockxs: {''uff(M(MNN({{ rc#Ktj|||j}|D]+\}}\}}\} } } ||dz z }| |dz z } ||||f| | f| fV,dS)z;Wraps a tokenize stream to systematically modify start/end.rN)rrc gen_lines__next__) rr9r<r;rAr,ruline0col0line1col1 line_texts rrNzRefactoringTool.wrap_tokss)$..*G*G*PQQDJ G G @D%% y VaZ E VaZ E t}udmYF F F F F G Grc#K||jz}||jz}|}|D]h}||r|t|dVn5||dzkrdVnt d|d||}i dV)zGenerates lines as expected by tokenize from a list of lines. This strips the first len(indent + self.PS1) characters off each line. Nrzline=z , prefix=Tr)r3r6rrr7AssertionError)rr9r;prefix1prefix2prefixr=s rrPzRefactoringTool.gen_liness 48#48#  Dv&& L3v;;<<(((((4/// $nTTT66%JKKKFF HHH rr)FF)F)NFNrU)r#r$r%rrrrrrrrrrrrrrrr rrr*r3r6rr4rKrArNrPr&rrr{r{s+0).2799LK3L3L3L3Ln&5&5&5P     FFFFLLLL(&&& ====.66666 M M M ^###.GL $BBBB** C C)))V:555. G G Grr{ceZdZdS)MultiprocessingUnsupportedNr"r&rrr]r]r'rr]cBeZdZfdZ dfd ZfdZfdZxZS)MultiprocessRefactoringToolcdtt|j|i|d|_d|_dSrU)superr_rqueue output_lockrrkwargsrs rrz$MultiprocessRefactoringTool.__init__s;9)40094J6JJJ rFrc|dkr*tt|||S ddln#t$rt wxYwjtd_ _ fdt|D} |D]}| tt|||j t|D]}jd|D]*}|r| +d_dS#j t|D]}jd|D]*}|r| +d_wxYw)Nrrz already doing multiple processescFg|]}jS))target)Process_child)rOr>multiprocessingrs rrRz8MultiprocessRefactoringTool.refactor..s<444%,,DK,@@444r)rar_rrk ImportErrorr]rb RuntimeError JoinableQueueLockrcrangestartrputis_alive) rrrr num_processes processesr6r>rkrs ` @rrz$MultiprocessRefactoringTool.refactors/ A  4d;;DDum-- - - " " " " " - - -, , - : !ABB B$2244 *//1144444#M22444     -t 4 4 = =eU>K M M M JOO   =)) % % t$$$$  ::<<FFHHHDJJJ JOO   =)) % % t$$$$  ::<<FFHHHDJ    s:A 4AE22A;G-c4|j}|{|\}} tt|j|i||jn#|jwxYw|j}|ydSdSrU)rbrrar_r task_done)rtaskrrers rrjz"MultiprocessRefactoringTool._childsz~~LD& 'F1488F%#%%% $$&&&& $$&&&&:>>##Ds AA8c|j|j||fdStt|j|i|SrU)rbrrrar_rrds rrz)MultiprocessRefactoringTool.refactor_filesV : ! JNND&> * * * * *I54d;;I!!! !r)FFr)r#r$r%rrrjr __classcell__)rs@rr_r_s     :? : $ $ $ $ $!!!!!!!!!rr_)T)#ry __author__rdrrr rrr9 itertoolsrpgen2rrr fixer_utilrrr r r rrr3r!r/rJrSrWrvrxobjectr{r]r_r&rrrs3   +*********!!!!!!            CCC82@@@%%%P''''''''FFFFFfFFFR        4!4!4!4!4!/4!4!4!4!4!r