+ i7Rt^RIt^RIHtHt^RIHtHt^RIHt^RI H t ^RI H t ^RI H t^RI Ht]P !R 4t.ROt!R R 4tR RltRRlt!RR 4tR#)z)[Experimental] Text Only Local Tokenizer.N)AnyIterable)OptionalUnion)sentencepiece_model_pb2)_common)_local_tokenizer_loader) _transformers)typeszgoogle_genai.local_tokenizerLocalTokenizer_TextsAccumulatorca]tRt^'toRtV3RlRltV3RlRltV3RlRltV3RlR ltV3R lR lt V3R lR lt V3RlRlt V3RlRlt V3RlRlt V3RlRltV3RlRltV3RlRltV3RlRltRtVtR#)r aAccumulates countable texts from `Content` and `Tool` objects. This class is responsible for traversing complex `Content` and `Tool` objects and extracting all the text content that should be included when calculating token counts. A key feature of this class is its ability to detect unsupported fields in `Content` objects. If a user provides a `Content` object with fields that this local tokenizer doesn't recognize (e.g., new fields added in a future API update), this class will log a warning. The detection mechanism for `Content` objects works by recursively building a "counted" version of the input object. This "counted" object only contains the data that was successfully processed and added to the text list for tokenization. After traversing the input, the original `Content` object is compared to the "counted" object. If they don't match, it signifies the presence of unsupported fields, and a warning is logged. c<V^8dQhRR/#)returnN)format __classdict__s"o/Users/igloo/.openclaw/workspace/scratch/fb_ad_env/lib/python3.14/site-packages/google/genai/local_tokenizer.py __annotate___TextsAccumulator.__annotate__;s   c.VnR#N_textsselfs&r__init___TextsAccumulator.__init__;s DKrc0<V^8dQhRS[S[,/#)rr)rstr)rrs"rrr>s#rcVP#rrrs&r get_texts_TextsAccumulator.get_texts>s ;;rcH<V^8dQhRS[S[P,RR/#)rcontentsrN)rr Content)rrs"rrrAs$  8EMM#: t rc:VFpVPV4K R#r) add_content)rr&contents&& r add_contents_TextsAccumulator.add_contentsAs wrc8<V^8dQhRS[PRR/#)rr*rN)r r')rrs"rrrEs4rc\P!.VPR7pVP'EdTVPEFBpVPfQh\P!4pVP fVP e \R4hVPeVPVnVPe-VPVP4VPVn VPe-VPVP4VPVn VPe7VPVn VPPVP4VPPV4EKE VP!RR7VP!RR78wd\"P%RV RV R24R#R#))partsroleNz6LocalTokenizers do not support non-text content types.T) exclude_nonezHContent contains unsupported types for token counting. Supported fields z. Got .)r r'r0r/Part file_data inline_data ValueErrorvideo_metadata function_calladd_function_callfunction_responseadd_function_responsetextrappend model_dumploggerwarning)rr*counted_contentpart counted_parts&& rr)_TextsAccumulator.add_contentEsmm"7<<@O}}}--$$$000zz| >> %)9)9)EF      *(,(;(;, %    )  !3!3 4'+'9'9, $  ! ! -  $ $T%;%; <+/+A+A, ( 99 "ii,  ++  TYY '$$\2% (t,0J0J1K1 nn $%VG9A 7rc8<V^8dQhRS[PRR/#)rr8rN)r FunctionCall)rrs"rrrds! 0 0U-?-? 0D 0rc VP'd&VPPVP4\P!VPR7pVP 'd$VP VP 4pW2nR#R#)z}Processes a function call and adds relevant text to the accumulator. Args: function_call: The function call to process. )nameN)rHrr=r rFargs_dict_traverse)rr8counted_function_call counted_argss&& rr9#_TextsAccumulator.add_function_calldsl  kk++,!..M4F4FG((););     kk-223*>*C*C"'''' kk-99:  * *#.&&&??+?+J+JK0B-$$$)=)F)FG.>+ ''rcN<V^8dQhRS[PRS[P/#)rschemar)r Schema)rrs"rrrs#++u||+ +rc &\P!4pVP'dVPVnVP'dVPVnVPeVPVnVP 'd7VP PVP 4VP VnVP'd7VP PVP4VPVnVP'd7VP PVP4VPVn VP'd7VP PVP4VPVn VP'dVPVn VP'd"VPVP4pW2n VP'd\/pVPP4F5wrVVP PV4VPV4pWtV&K7 WBnVP 'd"VP#VP 4pWnV#)zProcesses a schema and adds relevant text to the accumulator. Args: schema: The schema to process. Returns: The new schema object with only countable fields. )r rttypetitledefaultrrr=rmenumextendrequiredproperty_orderingitemsro propertiesexample _any_traverse) rrscounted_schemacounted_schema_itemsdkeyvalue counted_valuecounted_schema_examples && rro_TextsAccumulator.add_schemas\\^N {{{"KKn |||#\\n ~~!%~~n }}} kk'$mmn  kk++,#)#5#5n  {{{ kk%"KKn  kk) &n )/)A)An& |||!__V\\:1  a))//1*# 3. #2#$ ~~~#11&..A5 rcR<V^8dQhRS[S[S[3,RS[S[S[3,/#)rrr)dictr!r)rrs"rrrs+  d38n c3h rc /pVPP\VP444VP 4Fwr4VP V4W#&K V#)zProcesses a dict and adds relevant text to the accumulator. Args: d: The dict to process. Returns: The new dict object with only countable fields. )rrzlistkeysr}r)rr counted_dictrvals&& rrJ _TextsAccumulator._dict_traversesPLKKtAFFH~&GGI,,S1l rc&<V^8dQhRS[RS[/#)rrr)r)rrs"rrrsrc 0\V\4'dVPPV4V#\V\4'dVP V4#\V\ 4'd!VUu.uFq PV4NK up#V#uupi)zProcesses a value and adds relevant text to the accumulator. Args: value: The value to process. Returns: The new value with only countable fields. ) isinstancer!rr=rrJrr)rritems&& rr_TextsAccumulator._any_traverses{% kk l E4   '' E4 38 954  &5 99 l:s5BrN)__name__ __module__ __qualname____firstlineno____doc__rr#r+r)r9rWr\rbr;rSrorJr__static_attributes____classdictcell__rs@rr r 's&    > 0 0  44 < <(((++Z  rcxV^8dQhR\R\PPPR\ /#)rtokenrvr)r!r ModelProto SentencePieceTypebytes)rs"rrrs655 5-88FFKK5 5rcV\PPPP8Xd\ V4P ^RR7#VPRR4PR4#)big)length byteorderu▁ zutf-8) rrrrBYTE_parse_hex_byteto_bytesreplaceencode)rrvs&&r_token_str_to_bytesrs\ $ / / = = B B G GG 5 ! * *!u * EE == $ + +G 44rc0V^8dQhR\R\/#)rrr)r!int)rs"rrrs  3 3 rcF\V4^8wd\RV 24hVPR4'dVPR4'g\RV 24h\ VR,^4pTR8d\RT 24hT# \d\RT 24hi;i) zParses a hex byte string of the form '<0xXX>' and returns the integer value. Raises ValueError if the input is malformed or the byte value is invalid. zInvalid byte length: z<0x>zInvalid byte format: :NzInvalid hex value: zByte value out of range: )lenr6 startswithendswithr)rrs& rrrs  Z1_ ,UG4 55   % s(;(; ,UG4 554 eCj" C CZ 08 99 * 4 *5'2 334s BB ca]tRtRtoRtV3RlRlt]P!R4RR/V3RlR ll4t]P!R4V3R lR l4t R t Vt R#) r ia[Experimental] Text Only Local Tokenizer. This class provides a local tokenizer for text only token counting. LIMITATIONS: - Only supports text based tokenization and no multimodal tokenization. - Forward compatibility depends on the open-source tokenizer models for future Gemini versions. - For token counting of tools and response schemas, the `LocalTokenizer` only supports `types.Tool` and `types.Schema` objects. Python functions or Pydantic models cannot be passed directly. c <V^8dQhRS[/#)r model_name)r!)rrs"rrLocalTokenizer.__annotate__$sEEErc\P!V4Vn\P!VP4Vn\P !VP4VnR#r)loaderget_tokenizer_name_tokenizer_nameload_model_proto _model_protoget_sentencepiece _tokenizer)rrs&&rrLocalTokenizer.__init__$sF!44Z@D//0D0DED..t/C/CDDOrzThe SDK's local tokenizer implementation is experimental and may change in the future. It only supports text based tokenization.configNc<V^8dQhRS[S[PS[P3,RS[S[P ,RS[P /#)rr&rr)rr ContentListUnionContentListUnionDictrCountTokensConfigOrDictCountTokensResult)rrs"rrr-sP&&e,,e.H.HHI&u445 &   &rc \P!V4p\4p\PP T;'g/4pVP V4VP'dVPVP4VP'dBVPP'd&VPVPP4VP'd1VP \P!VP.44VPP\VP!444p\P"!\%RV44R7#)aCounts the number of tokens in a given text. Args: contents: The contents to tokenize. config: The configuration for counting tokens. Returns: A `CountTokensResult` containing the total number of tokens. Usage: .. code-block:: python from google import genai tokenizer = genai.LocalTokenizer(model_name='gemini-2.0-flash-001') result = tokenizer.count_tokens("What is your name?") print(result) # total_tokens=5 c38"TFp\V4xK R#5ir)r).0tokenss& r .LocalTokenizer.count_tokens..Rs?;V;s) total_tokens)t t_contentsr r CountTokensConfigmodel_validater+rZr\generation_configresponse_schemarosystem_instructionrrrr#rsum)rr&rprocessed_contentstext_accumulator tokens_lists&&$ r count_tokensLocalTokenizer.count_tokens)s :h/(*  $ $ 3 3FLLb AF!!"45 |||  . F$<$<$L$L$L!!&":":"J"JK    ##ALL&2K2K1L$MN//((.>.H.H.J)KLK  " "?;?? rcx<V^8dQhRS[S[PS[P3,RS[P/#)rr&r)rr rrComputeTokensResult)rrs"rrrYs;2>2>e,,e.H.HHI2>   2>rc \P!V4p\4pVFpVPV4K VPP VP 44p.pVFEpVP'gKVPFpVPVP4K KG .p\WV4FwrTP\P!V PU u.uFqPNK up V PU u.uFHp \V P VP"PV P,P$4NKJ up V R74K \P&!VR7#uup iuup i)aComputes the tokens ids and string pieces in the input. Args: contents: The contents to tokenize. Returns: A `ComputeTokensResult` containing the token information. Usage: .. code-block:: python from google import genai tokenizer = genai.LocalTokenizer(model_name='gemini-2.0-flash-001') result = tokenizer.compute_tokens("What is your name?") print(result) # tokens_info=[TokensInfo(token_ids=[279, 329, 1313, 2508, 13], tokens=[b' What', b' is', b' your', b' name', b'?'], role='user')] ) token_idsrr0) tokens_info)rrr r)rEncodeAsImmutableProtor#r/r=r0zipr TokensInfopiecesidrpiecerrvr) rr&rrr* tokens_protosroles_ token_infos tokens_protor0rs && rcompute_tokensLocalTokenizer.compute_tokensUsP4h/(*%""7+&OO::""$M E% A ,,w|| $& K!-7    /;/B/BC/Be/BC ,22  3e&kk4#4#4#;#;EHH#E#J#J 3    8  $ $ ==Ds ,FAF )rrr) rrrrrrrexperimental_warningrrrrrs@rr r sp EE  B&9= && &P B2> 2>r)rrr r )rloggingtypingrrrr sentencepiecerrrrr rr getLoggerr?__all__r rrr rrrrs] 0 "1/    9 : MM`5 ,u>u>r