How to adapt transfomer pretrained tokenizers to work with this translation tutorial?

The tutorial url: https://www.tensorflow.org/text/tutorials/transformer

The pt-en tokenizer model code:

examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True, as_supervised=True)
train_examples, val_examples = examples['train'], examples['validation']

# 2. Get BertTokenizer
model_name = "ted_hrlr_translate_pt_en_converter"
tf.keras.utils.get_file(
    f"{model_name}.zip",
    f"https://storage.googleapis.com/download.tensorflow.org/models/{model_name}.zip",
    cache_dir='.', cache_subdir='', extract=True
)

tokenizers = tf.saved_model.load(model_name)
en_tokenizer_items = [item for item in dir(tokenizers.en) if not item.startswith('_')]
print('En tokenizer methods: ', en_tokenizer_items)

# 3. Tokenizer examples
def tokenize_pairs(pt, en):
    pt = tokenizers.pt.tokenize(pt)
    
    # Convert from ragged to dense, padding with zeros.
    pt = pt.to_tensor()

    en = tokenizers.en.tokenize(en)
    # Convert from ragged to dense, padding with zeros.
    en = en.to_tensor()
    return pt, en


# 4. Make batches
BUFFER_SIZE = 20000
BATCH_SIZE = 64
def make_batches(ds):
  return (
      ds
      .cache()
      .shuffle(BUFFER_SIZE)
      .batch(BATCH_SIZE)
      .map(tokenize_pairs, num_parallel_calls=tf.data.experimental.AUTOTUNE)
      .prefetch(tf.data.experimental.AUTOTUNE))

train_batches = make_batches(train_examples)
val_batches = make_batches(val_examples)

In this line of code below, the tokenize takes a tensor of string as input:

pt = tokenizers.pt.tokenize(pt)

A transorformer pretrained tokenizer usually takes a string as input rather than a tensor here. If I want to switch the tokenizers from portugues to Chinese, how can I adapt the transformers tokenizer to work with the 'make_batches' and 'tokenize_pairs' functions?

I simply import the transformer tokenizers but it didn't work:

from transformers import BertTokenizer
tokenizer_en = BertTokenizer.from_pretrained("bert-base-cased")
tokenizer_zh = BertTokenizer.from_pretrained("bert-base-chinese")

def tokenize_pairs(zh, en):
    zh = tokenizer_zh.tokenize(zh)
    # Convert from ragged to dense, padding with zeros.
    zh = zh.to_tensor()
    en = tokenizer_en.tokenize(en)
    # Convert from ragged to dense, padding with zeros.
    en = en.to_tensor()
    return zh, en

This line below reports an error:

zh = tokenizer_zh.tokenize(zh)



/Users/cong/transformer/data_zh.py:44 tokenize_pairs  *
        zh = tokenizer_zh.tokenize(zh)
    /Users/cong/.venv/tf2/lib/python3.8/site-packages/transformers/tokenization_utils.py:336 split_on_tokens  *
        if not text.strip():
    /Users/cong/.venv/tf2/lib/python3.8/site-packages/tensorflow/python/framework/ops.py:401 __getattr__
        self.__getattribute__(name)

    AttributeError: 'Tensor' object has no attribute 'strip'