U 0JZdIdJZ?dKdLZ@GdMdNdNejjAjBZCdOdPZDGdQdRdRejjZEe"jdSdTfe"jdU<dVdWZFddYdZZGGd[d\d\ZHd]d^ZIdd`daZJdbdcZKdddeZLGdfdgdgZMddjdkZNGdldmdmZOdndoZPdpdqZQdrdsZRddtduZSGdvdwdwejjTjUZVe"jdxdyfe"jdz<d{d|ZWd}d~ZXdddZYddZZddZ[ddZ\dddZ]GdddejjTjUZ^GdddejjTjUZ_GdddejjZ`Gddde"j^ZadddZbGdddejjcjdZeddZfdddZgddZhdddZiddZjGdddejjTjUZkGdddejjTjUZlGddde"j_ZmGdddejjTjUZnddZoddZpGdddejjTjUZqGdddejjTjUZrGdddejjTjUZsGdddejjTjUZtGddde"j^ZuddZvdddZwddZxe"jddfe"jd<dddńZydddȄZzdddʄZ{Gdd̄d̃Z|dd΄Z}ddЄZ~dd҄ZddԄZddքZe"jddfe"jd<ddۄZejZejZejZejZejZejZejZejZejZejZejZejjZejjZejZejZejZejZejZejZejZejZejZejZejZejZejZddۄZdS)z,http://d2l-data.s3-accelerate.amazonaws.com/N) defaultdict)display)pyplot)backend_inlinecCstddS)zWUse the svg format to display a plot in Jupyter. Defined in :numref:`sec_calculus`svgN)rset_matplotlib_formatsrr=/home/d2l-worker/workspace/d2l-en-release/./d2l/tensorflow.pyuse_svg_display%sr g @@cCst|tjjd<dS)zJSet the figure size for matplotlib. Defined in :numref:`sec_calculus`zfigure.figsizeN)r d2lpltrcParamsfigsizerrr set_figsize+srcCsV|||||||||||||rJ|||dS)zCSet the axes for matplotlib. Defined in :numref:`sec_calculus`N) set_xlabel set_ylabel set_xscale set_yscaleset_xlimset_ylimlegendgrid)axesxlabelylabelxlimylimxscaleyscalerrrr set_axes2s       r"linear-zm--zg-.zr:c  Cs|dkr g}t| | r| ntj} dd} | |r<|g}|dkrZggt||}}n| |rh|g}t|t|kr|t|}| t||| D].\} }}t| r| | ||q| ||qt| |||||||dS)z8Plot data points. Defined in :numref:`sec_calculus`NcSs.t|dr|jdkp,t|to,t|dd S)Nndimr__len__)hasattrr& isinstancelistXrrr has_one_axisMszplot..has_one_axis) rr rgcalenclazipplotr")r-Yrrrrrr r!fmtsrrr.xyfmtrrr r3@s& r3c@s@eZdZdZddZddZddZdd Zd d Zd d Z dS)TimerzRecord multiple running times.cCsg|_|dS)z(Defined in :numref:`subsec_linear_model`N)timesstartselfrrr __init__cszTimer.__init__cCst|_dS)zStart the timer.N)timetikr<rrr r;hsz Timer.startcCs |jt|j|jdS)z-Stop the timer and record the time in a list.)r:appendr?r@r<rrr stoplsz Timer.stopcCst|jt|jS)zReturn the average time.)sumr:r0r<rrr avgqsz Timer.avgcCs t|jS)zReturn the sum of time.)rDr:r<rrr rDusz Timer.sumcCst|jS)zReturn the accumulated time.)nparrayr:cumsumtolistr<rrr rHysz Timer.cumsumN) __name__ __module__ __qualname____doc__r>r;rCrErDrHrrrr r9asr9cCsjt||jdf}|tjj|jd7}t|t|d|}|tjj|jdd7}t|d}||fS)zIGenerate y = Xw + b + noise. Defined in :numref:`sec_linear_scratch`rshaperAr'{Gz?)rOstddev)r zerosrOtfrandomnormalmatmulreshape)wb num_examplesr-r7rrr synthetic_data}s  r\cCst|||S)zIThe linear regression model. Defined in :numref:`sec_linear_scratch`)r rW)r-rYrZrrr linregsr]cCs|t||jddS)z:Squared loss. Defined in :numref:`sec_linear_scratch`)r rXrO)y_hatr7rrr squared_losssr`cCs*t||D]\}}||||q dS)zSMinibatch stochastic gradient descent. Defined in :numref:`sec_linear_scratch`N)r2Z assign_sub)paramsgradslr batch_sizeparamgradrrr sgdsrgTcCs,tjj|}|r|jdd}||}|S)zRConstruct a TensorFlow data iterator. Defined in :numref:`sec_linear_concise`) buffer_size)rTdataDatasetfrom_tensor_slicesshufflebatch) data_arraysrdis_traindatasetrrr load_arrays   rrc s*ddddddddd d g fd d |DS) z]Return text labels for the Fashion-MNIST dataset. Defined in :numref:`sec_fashion_mnist`zt-shirttrouserpulloverdresscoatsandalshirtsneakerbagz ankle bootcsg|]}t|qSr)int.0i text_labelsrr sz,get_fashion_mnist_labels..r)labelsrrr get_fashion_mnist_labelss r?c Cs||||f}tjj|||d\}}|}tt||D]N\}\} } | t| | j d| j d|r<| ||q<|S)zBPlot a list of images. Defined in :numref:`sec_fashion_mnist`rF) r rsubplotsflatten enumerater2imshownumpyr get_xaxis set_visible get_yaxis set_title) imgsnum_rowsnum_colstitlesscaler_rr~aximgrrr show_imagessrcsptjjj\}}dd}fdd}tjj||| t |d |tjj||| |fS)zlDownload the Fashion-MNIST dataset and then load it into memory. Defined in :numref:`sec_fashion_mnist`cSs tj|dddtj|ddfS)Naxisint32dtype)rT expand_dimscastr-r7rrr s z)load_data_fashion_mnist..csrtj|n||fSN)rTimageZresize_with_padrresizerr rsr) rTkerasdatasetsZ fashion_mnistZ load_datarjrkrlrnrmr0map)rdr mnist_train mnist_testprocess resize_fnrrr load_data_fashion_mnists   rcCsTt|jdkr*|jddkr*tj|dd}t||j|k}ttt||jS)zXCompute the number of correct predictions. Defined in :numref:`sec_softmax_scratch`r'r)r0rOr argmaxastyperfloat reduce_sum)r_r7cmprrr accuracysrcCsBtd}|D]$\}}|t|||t|q |d|dS)z\Compute the accuracy for a model on a dataset. Defined in :numref:`sec_softmax_scratch`r^rr') Accumulatoraddrr size)net data_itermetricr-r7rrr evaluate_accuracys rc@s0eZdZdZddZddZddZdd Zd S) rz)For accumulating sums over `n` variables.cCsdg||_dS)(Defined in :numref:`sec_softmax_scratch`Nrj)r=nrrr r>szAccumulator.__init__cGsddt|j|D|_dS)NcSsg|]\}}|t|qSr)r)r}arZrrr rsz#Accumulator.add..)r2rjr=argsrrr rszAccumulator.addcCsdgt|j|_dS)Nr)r0rjr<rrr resetszAccumulator.resetcCs |j|Srr)r=idxrrr __getitem__szAccumulator.__getitem__N)rJrKrLrMr>rrrrrrr rs rc Cs td}|D]\}}t4}||}t|tjjjrB|||} n |||} W5QRXt|tjjjr|j } | | | } | t | | n||j d| | |jt|tjjjr| tt|nt| } || t||t|q |d|d|d|dfS)zUThe training loop defined in Chapter 3. Defined in :numref:`sec_softmax_scratch`rrr^r')rrT GradientTaper*rlossesLoss optimizers Optimizertrainable_variablesgradientapply_gradientsr2rOrarrrrr) r train_iterlossupdaterrr-r7taper_lrarbZl_sumrrr train_epoch_ch3s(    rc @s"eZdZdZd ddZd d ZdS) AnimatorzFor plotting data in animation.Nr#r$r'r c szdkr gttjj| | | d\__| | dkrDjg_fdd_dd|___ dS)rNrr'c stjdSNr)r r"rrrr=rrr rrr!rr rsz#Animator.__init__..) r r rrfigr config_axesr-r4r5) r=rrrrrr r!r5nrowsncolsrrrr r>s  zAnimator.__init__cCst|ds|g}t|}t|ds,|g|}|jsFddt|D|_|js`ddt|D|_tt||D]<\}\}}|dk rn|dk rn|j|||j||qn|jd t|j|j|j D]\}}}|jd |||q| t |jt jdddS)Nr(cSsg|]}gqSrrr}rrrr r'sz Animator.add..cSsg|]}gqSrrrrrr r)srT)wait)r)r0r-ranger4rr2rBrr1r5r3rrr clear_output)r=r6r7rr~rrZr8rrr rs&    z Animator.add) NNNNNr#r#r$r'r'r )rJrKrLrMr>rrrrr r s rc Cstdd|gddgdddgd}t|D]2}t||||}t||} ||d|| fq&|\} } | d ksrt| | dkr| d kst| | dkr| d kst| d S) zSTrain a model (defined in Chapter 3). Defined in :numref:`sec_softmax_scratch`epochr'g333333?g? train loss train acctest acc)rrrr?gffffff?N)rrrrrAssertionError) rr test_iterr num_epochsranimatorr train_metricstest_acc train_loss train_accrrr train_ch35s  rc@s eZdZdZddZddZdS)UpdaterzrFor updating parameters using minibatch stochastic gradient descent. Defined in :numref:`sec_softmax_scratch`cCs||_||_dSr)rarc)r=rarcrrr r>HszUpdater.__init__cCst|j||j|dSr)r rgrarc)r=rdrbrrr __call__LszUpdater.__call__N)rJrKrLrMr>rrrrr rDsrcCs||D] \}}qqt|}ttj||dd}ddt||D}tjt|d||ddfd||d|ddS) zTPredict labels (defined in Chapter 3). Defined in :numref:`sec_softmax_scratch`r'rcSsg|]\}}|d|qS) r)r}truepredrrr rWszpredict_ch3..r)rN)r rrr2rrX)rrrr-r7truespredsrrrr predict_ch3Os   rcCsNtd}|D].\}}||||}|t|t|q|d|dS)z`Evaluate the loss of a model on the given dataset. Defined in :numref:`sec_model_selection`r^rr')r rrrr)rrrrr-r7rrrr evaluate_loss[s   rz..rjc Cs|tkst|dtdt|\}}tj|ddtj||dd}tj|rt }t |d"}| d}|s~q| |qnW5QRX| |kr|Std |d |d tj|ddd }t |d }||jW5QRX|S)zmDownload a file inserted into DATA_HUB, return the local filename. Defined in :numref:`sec_kaggle_house`z does not exist in .T)exist_ok/rArbiz Downloading z from z...)streamverifywb)DATA_HUBrosmakedirspathjoinsplitexistshashlibsha1openreadupdate hexdigestprintrequestsgetwritecontent) name cache_dirurl sha1_hashfnamerfrjrrrr downloadhs$      rcCs|t|}tj|}tj|\}}|dkr:t|d}n"|dkrPt|d}n ds\t d| ||rxtj ||S|S)zODownload and extract a zip/tar file. Defined in :numref:`sec_kaggle_house`z.zipr)z.tarz.gzFz$Only zip/tar files can be extracted.) rrrdirnamesplitextzipfileZipFiletarfilerr extractallr)rfolderrbase_dirdata_dirextfprrr download_extracts   r#cCstD] }t|qdS)zNDownload all files in the DATA_HUB. Defined in :numref:`sec_kaggle_house`N)rr)rrrr download_allsr$zkaggle_house_pred_train.csv(585e9cc93e70b39160e7921475f9bcd7d31219cekaggle_house_trainzkaggle_house_pred_test.csv(fa19780a7b011d9b009e8bff8e99922a8ee2eb90kaggle_house_testcCs4ttjjd|dkr*td|StdS)zVReturn gpu(i) if exists, otherwise return cpu(). Defined in :numref:`sec_use_gpu`GPUr'/GPU:/CPU:0)r0rTconfig experimentallist_physical_devicesdevice)r~rrr try_gpusr0cCs8ttjjd}ddt|D}|r,|StdgS)z^Return all available GPUs, or [cpu(),] if no GPU exists. Defined in :numref:`sec_use_gpu`r)cSsg|]}td|qS)r*)rTr/r|rrr rsz try_all_gpus..r+)r0rTr,r-r.rr/)num_gpusdevicesrrr try_all_gpussr3c Cs|j\}}tt|jd|d|jd|df}t|jdD]L}t|jdD]8}|||ft|||||||f|qZqH|S)zCompute 2D cross-correlation.rr')rOrTVariablerSrassignr)r-KhrYr4r~jrrr corr2ds 0 r9c@s*eZdZdZddZd ddZddZdS) TrainCallbackzSA callback to visiualize the training progress. Defined in :numref:`sec_lenet`cCsHt|_tjdd|gdddgd|_||_||_||_||_||_ dS)Nrr'rrr)rrr) r r9timerrrrrrr device_name)r=rrrrr<rrr r>s zTrainCallback.__init__NcCs|jdSr)r;r;)r=rlogsrrr on_epoch_beginszTrainCallback.on_epoch_begincCs|j|jj|jdddd}|d|d|f}|j|d|||jdkrtt |j dj d}|t j j|j }td|ddd |ddd |d dt||jd d t|jdS)NrT)verboseZ return_dictrrr'loss .3fz , train acc z , test acc r^.1fz examples/sec on )r;rCrevaluaterrrrnextiterrrOrTrjr- cardinalityrr rEstrr<)r=rr=rmetricsrdr[rrr on_epoch_ends"   ,zTrainCallback.on_epoch_end)N)rJrKrLrMr>r>rIrrrr r:s r:c Cs|j}tj|}|>tjjj|d}tjjj dd} |} | j || dgdW5QRXt | ||||} | j ||d| gd| S)zTTrain a model with a GPU (defined in Chapter 6). Defined in :numref:`sec_lenet` learning_rateT from_logitsr) optimizerrrHr)epochsr? callbacks) _device_namerT distributeZOneDeviceStrategyscoperrSGDrSparseCategoricalCrossentropycompiler:fit) Znet_fnrrrrcr/r<strategyrNrrcallbackrrr train_ch6s   rZcs*eZdZdZdfdd ZddZZS) ResidualzThe Residual block of ResNet.Fr'csxttjjj|dd|d|_tjjj|ddd|_d|_|rXtjjj|d|d|_tjj |_ tjj |_ dS)Nsamer)padding kernel_sizestrides)r^r]r')r^r_) superr>rTrlayersConv2Dconv1conv2conv3BatchNormalizationbn1bn2)r= num_channels use_1x1convr_ __class__rr r>s* zResidual.__init__cCsTtjj|||}|||}|jdk r>||}||7}tjj|Sr) rTr activationsrelurgrcrhrdre)r=r-r4rrr calls   z Residual.call)Fr'rJrKrLrMr>ro __classcell__rrrkr r[s r[ztimemachine.txt(090b5e7e70c295757f55df93cb0a180b9691891a time_machinec Cs2ttdd}|}W5QRXdd|DS)ziLoad the time machine dataset into a list of text lines. Defined in :numref:`sec_text_preprocessing`rsrcSs"g|]}tdd|qS)z [^A-Za-z]+ )resubstriplowerr}linerrr rsz%read_time_machine..)rr r readlines)rlinesrrr read_time_machinesr}wordcCs<|dkrdd|DS|dkr,dd|DStd|dS)z`Split text lines into word or character tokens. Defined in :numref:`sec_text_preprocessing`r~cSsg|] }|qSr)rryrrr rsztokenize..charcSsg|] }t|qSr)r+ryrrr rszERROR: unknown token type: N)r )r|tokenrrr tokenize s rc@sJeZdZdZdddZddZdd Zd d Zed d Z eddZ dS)VocabzVocabulary for text.NrcCs|dkr g}|dkrg}t|}t|dddd|_dg||_ddt|jD|_|jD]>\}}||krtq||jkr`|j|t|jd |j|<q`dS) z+Defined in :numref:`sec_text_preprocessing`NcSs|dSNr'r)r6rrr rz Vocab.__init__..T)keyreversezcSsi|]\}}||qSrr)r}rrrrr #sz"Vocab.__init__..r') count_corpussorteditems _token_freqs idx_to_tokenr token_to_idxrBr0)r=tokensmin_freqreserved_tokenscounterrfreqrrr r>s$   zVocab.__init__cCs t|jSr)r0rr<rrr r(,sz Vocab.__len__cs0t|ttfsj|jSfdd|DS)Ncsg|]}|qSr)r)r}rr<rr r2sz%Vocab.__getitem__..)r*r+tuplerr unk)r=rrr<r r/szVocab.__getitem__cs*t|ttfsj|Sfdd|DS)Ncsg|]}j|qSrr)r}indexr<rr r7sz#Vocab.to_tokens..)r*r+rr)r=indicesrr<r to_tokens4s zVocab.to_tokenscCsdSrrr<rrr r9sz Vocab.unkcCs|jSr)rr<rrr token_freqs=szVocab.token_freqs)NrN) rJrKrLrMr>r(rrpropertyrrrrrr rs  rcCs2t|dkst|dtr(dd|D}t|S)zICount token frequencies. Defined in :numref:`sec_text_preprocessing`rcSsg|]}|D]}|q qSrrr}rzrrrr rHsz count_corpus..)r0r*r+ collectionsCounter)rrrr rAsrrAcsFt}t|d}t|fdd|D}|dkr>|d|}|fS)zuReturn token indices and the vocabulary of the time machine dataset. Defined in :numref:`sec_text_preprocessing`rcsg|]}|D] }|q qSrrrvocabrr rTsz,load_corpus_time_machine..rN)r}rr) max_tokensr|rcorpusrrr load_corpus_time_machineKs  rc #stdddtd}ttd|}t|fdd||}td|||D]N}||||}fdd|D}fdd|D} t|t| fVqldS) zhGenerate a minibatch of subsequences using random sampling. Defined in :numref:`sec_language_model`rr'Ncs||Srr)pos)r num_stepsrr rjisz"seq_data_iter_random..datacsg|] }|qSrrr}r8rrr rrsz(seq_data_iter_random..csg|]}|dqS)r'rrrrr rss)rUrandintr0r+rrmr tensor) rrdr num_subseqsinitial_indices num_batchesr~initial_indices_per_batchr-r4r)rrjrr seq_data_iter_randomYs rc cstd|}t||d||}t||||}t||d|d|}t||df}t||df}|jd|}td|||D]>}|dd|||f} |dd|||f} | | fVqdS)zpGenerate a minibatch of subsequences using sequential partitioning. Defined in :numref:`sec_language_model`rr'rAN)rUrr0r rrXrOr) rrdroffset num_tokensXsYsrr~r-r4rrr seq_data_iter_sequentialvs rc@s eZdZdZddZddZdS) SeqDataLoaderz"An iterator to load sequence data.cCs:|rtj|_ntj|_t|\|_|_|||_|_dS)z'Defined in :numref:`sec_language_model`N) r r data_iter_fnrrrrrdr)r=rdruse_random_iterrrrr r>s  zSeqDataLoader.__init__cCs||j|j|jSr)rrrdrr<rrr __iter__szSeqDataLoader.__iter__N)rJrKrLrMr>rrrrr rs rF'cCst||||}||jfS)zpReturn the iterator and the vocabulary of the time machine dataset. Defined in :numref:`sec_language_model`)rr)rdrrrrrrr load_data_time_machinesrc@s(eZdZdZddZddZddZdS) RNNModelScratchz%A RNN Model implemented from scratch.cCs,|||_|_|||_|_||||_dS)z$Defined in :numref:`sec_rnn_scratch`N) vocab_size num_hiddens init_state forward_fnr)r=rrrr get_paramsrrr r>szRNNModelScratch.__init__cCs2tt||j}t|tj}||||jSr)rTone_hot transposerrfloat32rr)r=r-staterrr rszRNNModelScratch.__call__cOs|||jSr)rr)r=rdrkwargsrrr begin_stateszRNNModelScratch.begin_stateN)rJrKrLrMr>rrrrrr rsrcs|jdtjd}|dgfdd}|ddD]"}|||\}}|q6t|D]4}|||\}}t|jdddqbd fd d DS) zYGenerate new characters following the `prefix`. Defined in :numref:`sec_rnn_scratch`r'rdrrcsttdgdS)NrA)r'r')r rXrrr)outputsrr rrzpredict_ch8..Nrcsg|]}j|qSrrr|rrr rszpredict_ch8..) rrTrrBrr{rrrXr)prefix num_predsrrr get_inputr7rr)rrr predict_ch8s  "rcCstj|tjd}g}|D],}t|tjr:|t|q||qtjt dd|D}t |tj}t ||rt |D]\}}|||||<qn|}|S)zsz grad_clipping..) rTconstantrr*Z IndexedSlicesrBZconvert_to_tensormathsqrtrDrgreaterr)rbthetanew_gradrfnormr~rrr grad_clippings    rc Csdt}}td}|D]\}} |dks0|rF|j|jdtjd}tjdd0} |||\} }tt | d} || | } W5QRX|j }| | |}t |d}| t|||| t| t| qt|d|d|d|fS) z`Train a model within one epoch (defined in Chapter 8). Defined in :numref:`sec_rnn_scratch`Nr^rrT) persistentrAr')r r9rrrOrTrrrXrrrrrr2rrrexprC)rrrrrrr;rr-r4gr_r7rrarbrrr train_epoch_ch8s     rc s|$tjjjdd}tjj|}W5QRXtjdddgd|gd} fdd } t |D]F} t ||||\} } | d dd kr`t | d | | d | gq`t j}t d | dd| ddt|t | d t | ddS)zOTrain a model (defined in Chapter 8). Defined in :numref:`sec_rnn_scratch`TrLr perplexitytrain )rrrrcst|dS)N2)r)rrrrr rrztrain_ch8..r'rztime travellerz perplexity rB,  tokens/sec on travellerN)rSrTrrrUrrTr rrrr rr0rQrG)rrrrcrrXrrrrpredictrpplspeedr/rrr train_ch8s&     " rcs0eZdZdZfddZddZddZZS)RNNModelz$Defined in :numref:`sec_rnn-concise`c s2tt|jf|||_||_tjj||_ dSr) r`rr>rnnrrTrraDensedense)r= rnn_layerrrrkrr r>szRNNModel.__init__cCsHtt||j}|||^}}|t|d|jdf}||fS)NrA)rTrrrrrrXrO)r=inputsrr-r4outputrrr ro sz RNNModel.callcOs|jjj||Sr)rcellZget_initial_state)r=rrrrr rszRNNModel.begin_state)rJrKrLrMr>rorrqrrrkr rs rz fra-eng.zip(94646ad1522d915e7b0f9296181140edcf86a4f5fra-engc Cs>td}ttj|dd}|W5QRSQRXdS)zRLoad the English-French dataset. Defined in :numref:`sec_machine_translation`rzfra.txtrN)r r#rrrrr)r rrrr read_data_nmts rcsBddddddfddtD}d|S) zXPreprocess the English-French dataset. Defined in :numref:`sec_machine_translation`cSs|tdko|dkS)Nz,.!?rt)set)r prev_charrrr no_space%sz preprocess_nmt..no_spaceu rt cs6g|].\}}|dkr.||dr.d|n|qS)rr'rtr)r}r~rrtextrr r,sz"preprocess_nmt..r)replacerxrr)routrrr preprocess_nmt!s  rcCsxgg}}t|dD]V\}}|r0||kr0qp|d}t|dkr||dd||ddq||fS)zVTokenize the English-French dataset. Defined in :numref:`sec_machine_translation`r r^rrtr')rrr0rB)rr[sourcetargetr~rzpartsrrr tokenize_nmt0s    rcCstttjdd|Ddd|Dg\}}}tj|tj||djD]}|dqTtj|dS)z[Plot the histogram for list length pairs. Defined in :numref:`sec_machine_translation`cSsg|] }t|qSrr0r}rrrr rDsz+show_list_len_pair_hist..r'rN) r rrhistrrpatches set_hatchr)rrrxlistylistrrpatchrrr show_list_len_pair_hist>s    rcCs.t||kr|d|S||g|t|S)zLTruncate or pad sequences. Defined in :numref:`sec_machine_translation`Nr)rzr padding_tokenrrr truncate_padKs  rcsdfdd|D}fdd|D}tfdd|D}tt|dktjd}||fS)zrTransform text sequences of machine translation into minibatches. Defined in :numref:`subsec_mt_data_loading`csg|] }|qSrrrrrr rWsz#build_array_nmt..csg|]}|dgqS)rrrrr rXscsg|]}t|dqS))rrrrrr rYs r r')r rrrr)r|rrrG valid_lenrr r build_array_nmtSsr XcCstt}t||\}}tj|ddddgd}tj|ddddgd}t|||\}} t|||\} } || | | f} t| |} | ||fS)zuReturn the iterator and the vocabularies of the translation dataset. Defined in :numref:`subsec_mt_data_loading`r^r r)rr)rrrr rr rr)rdrr[rrr src_vocab tgt_vocab src_array src_valid_len tgt_array tgt_valid_lenrorrrr load_data_nmt_s   rcs(eZdZdZfddZddZZS)Encoderz@The base encoder interface for the encoder-decoder architecture.c stt|jf|dSr)r`rr>r=rrkrr r>qszEncoder.__init__cOstdSrNotImplementedError)r=r-rrrrr rotsz Encoder.callrprrrkr ros rcs0eZdZdZfddZddZddZZS)DecoderznThe base decoder interface for the encoder-decoder architecture. Defined in :numref:`sec_encoder-decoder`c stt|jf|dSr)r`rr>rrkrr r>{szDecoder.__init__cGstdSrr)r= enc_outputsrrrr r~szDecoder.init_statecKstdSrr)r=r-rrrrr rosz Decoder.call)rJrKrLrMr>rrorqrrrkr rws rcs(eZdZdZfddZddZZS)EncoderDecoderzbThe base class for the encoder-decoder architecture. Defined in :numref:`sec_encoder-decoder`c s"tt|jf|||_||_dSr)r`rr>encoderdecoder)r=rrrrkrr r>szEncoderDecoder.__init__cOs4|j|f||}|jj|f|}|j||f|Sr)rrr)r=enc_Xdec_Xrrr dec_staterrr roszEncoderDecoder.callrprrrkr rs rcs*eZdZdZdfdd ZddZZS)Seq2SeqEncoderzXThe RNN encoder for sequence to sequence learning. Defined in :numref:`sec_seq2seq`rc sVtj|tjj|||_tjjjtjjfddt |Dddd|_ dS)Ncsg|]}tjjjdqS))dropout)rTrraGRUCellrr#rrr rsz+Seq2SeqEncoder.__init__..T)Zreturn_sequencesZ return_state) r`r>rTrra Embedding embeddingRNNZStackedRNNCellsrr)r=r embed_sizer num_layersr#rrkr%r r>s  zSeq2SeqEncoder.__init__cOs0||}|j|f|}|dd}|d|fS)Nr'r)r'r)r=r-rrrrrrr ros  zSeq2SeqEncoder.call)rrprrrkr r"s r"cCs||jd}tjd|tjddddftj|dddftjdk}t|jdkrjttj|dd||St|||SdS) zSMask irrelevant entries in sequences. Defined in :numref:`sec_seq2seq_decoder`r'r)r;limitrNrrrAr)rOrTrrrr0wherer)r-r valuemaxlenmaskrrr sequence_masks  r0cs(eZdZdZfddZddZZS)MaskedSoftmaxCELosszXThe softmax cross-entropy loss with masks. Defined in :numref:`sec_seq2seq_decoder`cstjdd||_dS)Nnone) reduction)r`r>r )r=r rkrr r>szMaskedSoftmaxCELoss.__init__cCs^tj|tjd}t||j}tj||jdd}tjjj ddd||}tj ||dd}|S) NrrA)depthTr2)rMr3r'r) rT ones_likerr0r rrOrrZCategoricalCrossentropy reduce_mean)r=labelrweightsZ label_one_hotunweighted_loss weighted_lossrrr ros zMaskedSoftmaxCELoss.callrprrrkr r1s r1c Cstjjj|d}tjddd|gd}t|D]*}t} td} |D]} dd| D\} } }}tj t |d g|j d d d }t ||d d d dfgd}t (}|| || dd\}}t|||}W5QRX|||j}t|d}|t||jt|}| t||qH|ddd kr,||d| d | dfq,td| d | ddd| d| ddt|d S)zUTrain a model for sequence to sequence. Defined in :numref:`sec_seq2seq_decoder`rJrrr)rrrr^cSsg|]}|qSrr)r}r6rrr rsz!train_seq2seq..rrrPrNNrAr'Ttrainingr@rArrBr)rTrrAdamr rrr9rrXrrOconcatrr1rrrrr2rrrr rCrG)rrrcrrr/rNrrr;rrnr- X_valid_lenr4 Y_valid_lenbos dec_inputrY_hatrr gradientsrrrr train_seq2seqs0     rEcCs"||d|dg}tt|g}t|||d}tj|dd}|j||dd} |j | |} tjt|dgdd} gg} } t |D]d}|j | | dd\}} tj |d d} tj | dd}|r| |j j||dkrq| |qd|tj| d d | fS) zPPredict for sequence to sequence. Defined in :numref:`sec_seq2seq_training`rtrr rrFr;rr^rArN)rxrrTrr0r rrrrrrrsqueezerBattention_weightsrrrrXrI)r src_sentencerrrsave_attention_weights src_tokens enc_valid_lenrrr!r  output_seqattention_weight_seqrr4rrrr predict_seq2seqs(   rNc Cs"|d|d}}t|t|}}ttdd||}td|dD]}dtt} } t||dD]&} | d || | |d7<qrt||dD]L} | d || | |dkr| d7} | d || | |d8<q|t | ||dt d|9}qN|S)z@Compute the BLEU. Defined in :numref:`sec_seq2seq_training`rtrr'r) rr0rrminrrrr{rpow) pred_seq label_seqk pred_tokens label_tokenslen_pred len_labelscorer num_matches label_subsr~rrr bleus$$&r[r r RedscCst|jd|jd}}tjj|||dddd\}} tt| |D]v\} \} } tt| | D]Z\} \}}|jt||d}| |dkr| || dkr| ||rb| || qbqH|j || ddd S) zGShow heatmaps of matrices. Defined in :numref:`sec_attention-cues`rr'TF)rsharexshareyrF)cmapg333333?)rshrinkN) r r rOrrrr2rrrrrcolorbar)matricesrrrrr`rrrrr~row_axes row_matricesr8rmatrixpcmrrr show_heatmapss"     rhcCs|dkrtjj|ddS|j}t|jdkr@tj||dd}ntj|dd}tjtj|d|dfd|dd}tjjtj||dddSdS) zyPerform softmax operation by masking elements on the last axis. Defined in :numref:`sec_attention-scoring-functions`NrArr')repeatsrNg.)r-) rTnnsoftmaxrOr0repeatrXr r0)r- valid_lensrOrrr masked_softmax*s"rncs(eZdZdZfddZddZZS)AdditiveAttentionzMAdditive attention. Defined in :numref:`sec_attention-scoring-functions`c s^tjf|tjjj|dd|_tjjj|dd|_tjjjddd|_tjj ||_ dS)NFuse_biasr') r`r>rTrrarW_kW_qw_vDropoutr#)r=key_size query_sizerr#rrkrr r>As zAdditiveAttention.__init__cKsv||||}}tj|ddtj|dd}tj|}tj||dd}t|||_ t |j |j f||S)Nr^rr'rA) rsrrrTrrjtanhrFrtrnrGrWr#)r=querieskeysvaluesrmrfeaturesscoresrrr roHs  zAdditiveAttention.callrprrrkr ro=s rocs(eZdZdZfddZddZZS)DotProductAttentionzQScaled dot product attention. Defined in :numref:`subsec_additive-attention`c s"tjf|tjj||_dSr)r`r>rTrrarur#)r=r#rrkrr r>_szDotProductAttention.__init__cKsV|jd}tj||ddtjtj|tjd}t|||_t|j |jf||S)NrAT) transpose_br) rOrTrWrrrrrnrGr#)r=ryrzr{rmrdr}rrr rohs   zDotProductAttention.callrprrrkr r~[s r~cs,eZdZdZfddZeddZZS)AttentionDecoderz[The base attention-based decoder interface. Defined in :numref:`sec_seq2seq_attention`c stt|jf|dSr)r`rr>rrkrr r>sszAttentionDecoder.__init__cCstdSrrr<rrr rGvsz"AttentionDecoder.attention_weights)rJrKrLrMr>rrGrqrrrkr ros rcs*eZdZdZdfdd ZddZZS)MultiHeadAttentionzGMulti-head attention. Defined in :numref:`sec_multihead-attention`Fc  sttjf|||_t||_tjjj ||d|_ tjjj ||d|_ tjjj ||d|_ tjjj ||d|_ dS)Nrp)r`r> num_headsr r~ attentionrTrrarrsrrW_vW_o) r=rvrw value_sizerrr#biasrrkrr r>~s zMultiHeadAttention.__init__cKszt|||j}t|||j}t|||j}|dk rPtj||jdd}|j||||f|}t||j}| |S)Nr)rir) transpose_qkvrsrrrrrTrlrtranspose_outputr)r=ryrzr{rmrr output_concatrrr ros zMultiHeadAttention.call)Frprrrkr rzs rcCsPtj||jd|jd|dfd}tj|dd}tj|d|jd|jdfdS) zuTransposition for parallel computation of multiple attention heads. Defined in :numref:`sec_multihead-attention`rr'rArNrr^r'rpermr^rrTrXrOrr-rrrr rs "rcCsPtj|d||jd|jdfd}tj|dd}tj||jd|jddfdS)z[Reverse the operation of `transpose_qkv`. Defined in :numref:`sec_multihead-attention`rAr'r^rNrrrrrrrr rs"rcs*eZdZdZdfdd ZddZZS)PositionalEncodingzYPositional encoding. Defined in :numref:`sec_self-attention-and-positional-encoding`rhc sttjj||_td||f|_ tj |tj d ddt dtj d|dtj d|}t||j dddddddf<t||j dddddddf<dS)Nr'rrArrr^)r`r>rTrrarur#rFrSParangerrXpowersincos)r=rr#max_lenr-rkrr r>s$  $zPositionalEncoding.__init__cKs4||jddd|jdddf}|j|f|Sr)rrOr#)r=r-rrrr ros&zPositionalEncoding.call)rhrprrrkr rs rcs(eZdZdZfddZddZZS)PositionWiseFFNzLPositionwise feed-forward network. Defined in :numref:`sec_transformer`c s>tj|tjj||_tjj|_tjj||_ dSr) r`r>rTrrardense1ReLUrndense2)r=ffn_num_hiddensffn_num_outputsrrkrr r>s zPositionWiseFFN.__init__cCs||||Sr)rrnr)r=r-rrr roszPositionWiseFFN.callrprrrkr rs rcs(eZdZdZfddZddZZS)AddNormz^Residual connection followed by layer normalization. Defined in :numref:`sec_transformer`c s2tjf|tjj||_tjj||_dSr) r`r>rTrrarur#ZLayerNormalizationln)r=normalized_shaper#rrkrr r>szAddNorm.__init__cKs||j|f||Sr)rr#)r=r-r4rrrr rosz AddNorm.callrprrrkr rs rcs*eZdZdZdfdd ZddZZS) EncoderBlockzDTransformer encoder block. Defined in :numref:`sec_transformer`Fc sNtjf| t||||||| |_t|||_t|||_t|||_ dSr) r`r>r rrraddnorm1rffnaddnorm2) r=rvrwrr norm_shaperrr#rrrkrr r>s   zEncoderBlock.__init__cKs6|j||j||||f|f|}|j|||f|Sr)rrrr)r=r-rmrr4rrr ros zEncoderBlock.call)Frprrrkr rs rcs*eZdZdZdfdd ZddZZS)TransformerEncoderz>Transformer encoder. Defined in :numref:`sec_transformer`Fc s`tjf| |_tjj||_t |_ f ddt | D|_ dS)Nc s$g|]}t qSr)rr rr#rrvrrrrwrrr rsz/TransformerEncoder.__init__..) r`r>rrTrrar&r'r r pos_encodingrblks) r=rrvrwrrrrrr*r#rrrkrr r>szTransformerEncoder.__init__cKsv|j||tjtj|jtjdf|}dgt|j |_ t |j D]&\}}|||f|}|j j j |j |<qJ|S)Nr) rr'rTrrrrrr0rrGrr)r=r-rmrr~blkrrr ro szTransformerEncoder.call)Frprrrkr rs rcCs"tjj|||tddddS)Nz->) arrowstyle)xyxytext arrowprops)r rr/annotatedict)rrrrrr rsrc Csd\}}}}||fg}t|D]F}|r@||||||\}}}}n|||||\}}}}|||fqtd|ddt|ddt|d|S)zlOptimize a 2D objective function with a customized trainer. Defined in :numref:`subsec_gd-learningrate`)rrzepoch r'z, x1: rz, x2: )rrBr r) trainerstepsf_gradx1x2s1s2resultsr~rrr train_2ds   *rcCs|ttjjt|d ddittdddtddd\}}tjj|||||dd tjd tj d d S)zdShow the trace of 2D variables during optimization. Defined in :numref:`subsec_gd-learningrate`-ocolorz#ff7f0egg?g?gz#1f77b4)colorsrrN)r) r rrr3r2meshgridrcontourrr)rrrrrrr show_trace_2d,s  rzairfoil_self_noise.dat(76e5be1548fd8222e5074cf0faae75edff8cf93fairfoilrcCsvtjtdtjdd}||jdd|jdd}tj|d|ddf|d|dff|dd }||jd d fS) $Defined in :numref:`sec_minibatches`rr)r delimiterrrNrAT)rpr') rF genfromtxtr rrmeanstdrrrO)rdrrjrrrr get_data_ch11;s &rr^c stjtjj|dfdddddtjtdddfddtj}}tjd d d|gd d gd }dt} } t |D]} |D]\} } t }tj ||| | }W5QRX| |g\}}|g||g||| | jd7} | ddkr| | | jd}|tjj|}t|||f}|||| qqztd|jdddd| dd| |jdfS)rr'rrQ)rOrrRT) trainablecst|Sr)r r]r,rZrYrr rMrztrain_ch11..rr)\(?ffffff?rrrrloss: rArAr sec/epoch)rTr4rUrVrSr r`rr9rrrr6rrOrCrjr-rFrrrr;r r4rErH) trainer_fnstates hyperparamsr feature_dimrrrrrr;rr-r7rrdwdbpqrrrr train_ch11Ds:       (rc Csftj}|tjjjdtjddd|f|}tjj}t j ddd|gdd gd }dt }} t |D]} |D]\} } t *} || }|| |}|j}| ||}W5QRX|t|||| jd7}|d dkrx| || jd}|tjj|}t |||d f}|||| qxqptd |jdddd| dddS)rr'rQ)rR)Zkernel_initializerrrrrrrrr^rrArArrN)rTr SequentialrrarZrandom_normal_initializerrZMeanSquaredErrorr rr9rrrrrr2rOrCrjr-rFrrr;r r4rE)rrrrrrNrrrr;rr-r7rrrrarbrrrrrr train_concise_ch11ds8           rc@s*eZdZdZd ddZddZddZd S) BenchmarkzFor measuring running time.DonecCs ||_dS)z"Defined in :numref:`sec_hybridize`N) description)r=rrrr r>szBenchmark.__init__cCst|_|Sr)r r9r;r<rrr __enter__s zBenchmark.__enter__cGs"t|jd|jdddS)Nz: z.4fz sec)r rr;rCrrrr __exit__szBenchmark.__exit__N)r)rJrKrLrMr>rrrrrr rs rc Cs|dddf|dddf|dddf|dddff\}}}}||d}||d}||}||}tj||||fdd}|S)zeConvert from (upper-left, lower-right) to (center, width, height). Defined in :numref:`sec_bbox`Nrr'r^rrArr stack) boxesry1ry2cxcyrYr7rrr box_corner_to_centersD  rc Cs|dddf|dddf|dddf|dddff\}}}}|d|}|d|}|d|}|d|}tj||||fdd}|S) zeConvert from (center, width, height) to (upper-left, lower-right). Defined in :numref:`sec_bbox`Nrr'r^rrrArr) rrrrYr7rrrrrrr box_center_to_cornersD    rcCs<tjj|d|df|d|d|d|dd|ddS)zMConvert bounding box to matplotlib format. Defined in :numref:`sec_bbox`rr'r^rF)rwidthheightfill edgecolor linewidth)r r Rectangle)bboxrrrr bbox_to_rectsrc Cs|jd}t|f}t|f}||} t>} ||} || } ||t| ||t| |d} W5QRX| | |j}|t ||j| S)z=Update discriminator. Defined in :numref:`sec_basic_gan`rr^) rOrTonesrSrrFrrrr2)r-Znet_Dnet_GrZ optimizer_DrdrrSfake_Xrreal_Yfake_Yloss_DZgrads_Drrr update_Ds$    rc Csr|jd}t|f}t*}||}||} ||t| |} W5QRX|| |j} |t| |j| S)z9Update generator. Defined in :numref:`sec_basic_gan`r) rOrTrrrFrrrr2) rrrrZ optimizer_Grdrrrrloss_GZgrads_Grrr update_Gs   rz pokemon.zip(c065c0e2593b8b161a2d7873e42418bf6a21106cpokemoncCst|Sr)rTrr)rrrr rrrcOs |j||Sr)r)r6rrrrr rr)r ) NNNNNNr#r#r$r N)T)Nr)N)r)N)r)r~)rA)Fr)F)N)r )r)F)Nr\r])rN)rr)r^)r^)rrDATA_URLrrF tensorflowrTrModel nn_ModulerrrrrUrushutilsysrr?rrpandaspdr IPythonr matplotlibrrmatplotlib_inlinermodulesrJr r rr"r3r9r\r]r`rgrrrrrrrrrrrrrrrrrr#r$r0r3r9rPZCallbackr:rZr[r}rrrrrrrrrrrrrraZLayerrrrrrrr rrrrr"r0rrr1rErNr[rhrnror~rrrrrrrrrrrrrrrrrrrrrrrXrrSrrsinhrcoshrxlinspacerrVuniformrandrWrrrrrrrrrrrr>rabseyerrrr sZ       !      )       ,            &