|
| 1 | +import logging |
| 2 | +import time |
| 3 | +from abc import ABC |
| 4 | + |
| 5 | +import requests |
| 6 | +import torch |
| 7 | +import transformers |
| 8 | +from transformers import AutoModelForCausalLM, AutoTokenizer |
| 9 | + |
| 10 | +from ts.handler_utils.distributed.pt_pippy import get_pipeline_driver |
| 11 | +from ts.torch_handler.distributed.base_pippy_handler import BasePippyHandler |
| 12 | + |
| 13 | +logger = logging.getLogger(__name__) |
| 14 | +logger.info("Transformers version %s", transformers.__version__) |
| 15 | + |
| 16 | + |
| 17 | +class TransformersSeqClassifierHandler(BasePippyHandler, ABC): |
| 18 | + """ |
| 19 | + Transformers handler class for sequence, token classification and question answering. |
| 20 | + """ |
| 21 | + |
| 22 | + def __init__(self): |
| 23 | + super(TransformersSeqClassifierHandler, self).__init__() |
| 24 | + self.initialized = False |
| 25 | + |
| 26 | + def initialize(self, ctx): |
| 27 | + """In this initialize function, the HF large model is loaded and |
| 28 | + partitioned into multiple stages each on one device using PiPPy. |
| 29 | + Args: |
| 30 | + ctx (context): It is a JSON Object containing information |
| 31 | + pertaining to the model artefacts parameters. |
| 32 | + """ |
| 33 | + super().initialize(ctx) |
| 34 | + self.manifest = ctx.manifest |
| 35 | + properties = ctx.system_properties |
| 36 | + model_dir = properties.get("model_dir") |
| 37 | + self.device = self.local_rank |
| 38 | + |
| 39 | + seed = ctx.model_yaml_config["handler"]["manual_seed"] |
| 40 | + torch.manual_seed(seed) |
| 41 | + |
| 42 | + self.model = AutoModelForCausalLM.from_pretrained(model_dir, use_cache=False) |
| 43 | + |
| 44 | + self.tokenizer = AutoTokenizer.from_pretrained(model_dir, return_tensors="pt") |
| 45 | + |
| 46 | + self.max_length = ctx.model_yaml_config["handler"]["max_length"] |
| 47 | + |
| 48 | + logger.info("Instantiating model Pipeline") |
| 49 | + model_init_start = time.time() |
| 50 | + self.model = get_pipeline_driver(self.model, self.world_size, ctx) |
| 51 | + |
| 52 | + logger.info("Transformer model from path %s loaded successfully", model_dir) |
| 53 | + |
| 54 | + self.initialized = True |
| 55 | + |
| 56 | + def preprocess(self, requests): |
| 57 | + """ |
| 58 | + Basic text preprocessing, based on the user's choice of application mode. |
| 59 | + Args: |
| 60 | + requests (list): A list of dictionaries with a "data" or "body" field, each |
| 61 | + containing the input text to be processed. |
| 62 | + Returns: |
| 63 | + tuple: A tuple with two tensors: the batch of input ids and the batch of |
| 64 | + attention masks. |
| 65 | + """ |
| 66 | + input_texts = [data.get("data") or data.get("body") for data in requests] |
| 67 | + input_ids_batch, attention_mask_batch = [], [] |
| 68 | + for input_text in input_texts: |
| 69 | + input_ids, attention_mask = self.encode_input_text(input_text) |
| 70 | + input_ids_batch.append(input_ids) |
| 71 | + attention_mask_batch.append(attention_mask) |
| 72 | + input_ids_batch = torch.cat(input_ids_batch, dim=0).to(self.device) |
| 73 | + attention_mask_batch = torch.cat(attention_mask_batch, dim=0).to(self.device) |
| 74 | + return input_ids_batch, attention_mask_batch |
| 75 | + |
| 76 | + def encode_input_text(self, input_text): |
| 77 | + """ |
| 78 | + Encodes a single input text using the tokenizer. |
| 79 | + Args: |
| 80 | + input_text (str): The input text to be encoded. |
| 81 | + Returns: |
| 82 | + tuple: A tuple with two tensors: the encoded input ids and the attention mask. |
| 83 | + """ |
| 84 | + if isinstance(input_text, (bytes, bytearray)): |
| 85 | + input_text = input_text.decode("utf-8") |
| 86 | + logger.info("Received text: '%s'", input_text) |
| 87 | + inputs = self.tokenizer.encode_plus( |
| 88 | + input_text, |
| 89 | + max_length=self.max_length, |
| 90 | + pad_to_max_length=True, |
| 91 | + add_special_tokens=True, |
| 92 | + return_tensors="pt", |
| 93 | + ) |
| 94 | + input_ids = inputs["input_ids"] |
| 95 | + attention_mask = inputs["attention_mask"] |
| 96 | + return input_ids, attention_mask |
| 97 | + |
| 98 | + def inference(self, input_batch): |
| 99 | + """ |
| 100 | + Predicts the class (or classes) of the received text using the serialized transformers |
| 101 | + checkpoint. |
| 102 | + Args: |
| 103 | + input_batch (tuple): A tuple with two tensors: the batch of input ids and the batch |
| 104 | + of attention masks, as returned by the preprocess function. |
| 105 | + Returns: |
| 106 | + list: A list of strings with the predicted values for each input text in the batch. |
| 107 | + """ |
| 108 | + input_ids_batch, attention_mask_batch = input_batch |
| 109 | + input_ids_batch = input_ids_batch.to(self.device) |
| 110 | + outputs = self.model.generate( |
| 111 | + input_ids_batch, |
| 112 | + attention_mask=attention_mask_batch, |
| 113 | + max_length=30, |
| 114 | + ) |
| 115 | + |
| 116 | + inferences = [ |
| 117 | + self.tokenizer.batch_decode( |
| 118 | + outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False |
| 119 | + ) |
| 120 | + ] |
| 121 | + logger.info("Generated text: %s", inferences) |
| 122 | + return inferences |
| 123 | + |
| 124 | + def postprocess(self, inference_output): |
| 125 | + """Post Process Function converts the predicted response into Torchserve readable format. |
| 126 | + Args: |
| 127 | + inference_output (list): It contains the predicted response of the input text. |
| 128 | + Returns: |
| 129 | + (list): Returns a list of the Predictions and Explanations. |
| 130 | + """ |
| 131 | + return inference_output |
0 commit comments