Serverless BERT with HuggingFace and AWS Lambda

Copy 
1context = """We introduce a new language representation model called BERT, which stands for
2Bidirectional Encoder Representations from Transformers. Unlike recent language
3representation models (Peters et al., 2018a; Radford et al., 2018), BERT is
4designed to pretrain deep bidirectional representations from unlabeled text by
5jointly conditioning on both left and right context in all layers. As a result,
6the pre-trained BERT model can be finetuned with just one additional output
7layer to create state-of-the-art models for a wide range of tasks, such as
8question answering and language inference, without substantial taskspecific
9architecture modifications. BERT is conceptually simple and empirically
10powerful. It obtains new state-of-the-art results on eleven natural language
11processing tasks, including pushing the GLUE score to 80.5% (7.7% point absolute
12improvement), MultiNLI accuracy to 86.7% (4.6% absolute improvement), SQuAD v1.1
13question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD
14v2.0 Test F1 to 83.1 (5.1 point absolute improvement)."""
15
16question_one = "What is BERTs best score on Squadv2 ?"
17# 83 . 1
18
19question_two = "What does the 'B' in BERT stand for?"
20# 'bidirectional encoder representations from transformers'

Copy 
1serverless create --template aws-python3 --path serverless-bert

Copy 
1import json
2
3def hello(event, context):
4    body = {
5        "message": "Go Serverless v1.0! Your function executed successfully!",
6        "input": event
7    }
8    response = {
9        "statusCode": 200,
10        "body": json.dumps(body)
11    }
12    return response

Copy 
1https://download.pytorch.org/whl/cpu/torch-1.5.0%2Bcpu-cp38-cp38-linux_x86_64.whl
2transformers==2.10

Copy 
1aws s3api create-bucket --bucket bucket-name --region eu-central-1 --create-bucket-configuration LocationConstraint=eu-central-1

Copy 
1import boto3
2
3def upload_model(model_path='', s3_bucket='', key_prefix='', aws_profile='default'):
4    s3 = boto3.session.Session(profile_name=aws_profile)
5    client = s3.client('s3')
6    client.upload_file(model_path, s3_bucket, key_prefix)

Copy 
1service: serverless-bert
2
3provider:
4  name: aws
5  runtime: python3.8
6  region: eu-central-1
7  timeout: 60
8  iamRoleStatements:
9    - Effect: "Allow"
10      Action:
11        - s3:getObject
12      Resource: arn:aws:s3:::<your-S3-Bucket>/<key_prefix>/*
13
14custom:
15  pythonRequirements:
16    dockerizePip: true
17    zip: true
18    slim: true
19    strip: false
20    noDeploy:
21      - docutils
22      - jmespath
23      - pip
24      - python-dateutil
25      - setuptools
26      - six
27      - tensorboard
28    useStaticCache: true
29    useDownloadCache: true
30    cacheLocation: "./cache"
31package:
32  individually: false
33  exclude:
34    - package.json
35    - package-log.json
36    - node_modules/**
37    - cache/**
38    - test/**
39    - __pycache__/**
40    - .pytest_cache/**
41    - model/pytorch_model.bin
42    - raw/**
43    - .vscode/**
44    - .ipynb_checkpoints/**
45
46functions:
47  predict_answer:
48    handler: handler.predict_answer
49    memorySize: 3008
50    timeout: 60
51    events:
52      - http:
53          path: ask
54          method: post
55          cors: true
56
57plugins:
58  - serverless-python-requirements

Copy 
1from transformers import AutoModelForQuestionAnswering, AutoTokenizer, AutoConfig
2import torch
3import boto3
4import os
5import tarfile
6import io
7import base64
8import json
9import re
10
11s3 = boto3.client('s3')
12
13class ServerlessModel:
14    def __init__(self, model_path=None, s3_bucket=None, file_prefix=None):
15        self.model, self.tokenizer = self.from_pretrained(
16            model_path, s3_bucket, file_prefix)
17
18    def from_pretrained(self, model_path: str, s3_bucket: str, file_prefix: str):
19        model = self.load_model_from_s3(model_path, s3_bucket, file_prefix)
20        tokenizer = self.load_tokenizer(model_path)
21        return model, tokenizer
22
23    def load_model_from_s3(self, model_path: str, s3_bucket: str, file_prefix: str):
24        if model_path and s3_bucket and file_prefix:
25            obj = s3.get_object(Bucket=s3_bucket, Key=file_prefix)
26            bytestream = io.BytesIO(obj['Body'].read())
27            tar = tarfile.open(fileobj=bytestream, mode="r:gz")
28            config = AutoConfig.from_pretrained(f'{model_path}/config.json')
29            for member in tar.getmembers():
30                if member.name.endswith(".bin"):
31                    f = tar.extractfile(member)
32                    state = torch.load(io.BytesIO(f.read()))
33                    model = AutoModelForQuestionAnswering.from_pretrained(
34                        pretrained_model_name_or_path=None, state_dict=state, config=config)
35            return model
36        else:
37            raise KeyError('No S3 Bucket and Key Prefix provided')
38
39    def load_tokenizer(self, model_path: str):
40        tokenizer = AutoTokenizer.from_pretrained(model_path)
41        return tokenizer
42
43    def encode(self, question, context):
44        encoded = self.tokenizer.encode_plus(question, context)
45        return encoded["input_ids"], encoded["attention_mask"]
46
47    def decode(self, token):
48        answer_tokens = self.tokenizer.convert_ids_to_tokens(
49            token, skip_special_tokens=True)
50        return self.tokenizer.convert_tokens_to_string(answer_tokens)
51
52    def predict(self, question, context):
53        input_ids, attention_mask = self.encode(question, context)
54        start_scores, end_scores = self.model(torch.tensor(
55            [input_ids]), attention_mask=torch.tensor([attention_mask]))
56        ans_tokens = input_ids[torch.argmax(
57            start_scores): torch.argmax(end_scores)+1]
58        answer = self.decode(ans_tokens)
59        return answer

Copy 
1try:
2    import unzip_requirements
3except ImportError:
4    pass
5from model.model import ServerlessModel
6import json
7
8model = ServerlessModel('./model', <s3_bucket>, <file_prefix>)
9
10def predict_answer(event, context):
11    try:
12        body = json.loads(event['body'])
13        answer = model.predict(body['question'], body['context'])
14
15        return {
16            "statusCode": 200,
17            "headers": {
18                'Content-Type': 'application/json',
19                'Access-Control-Allow-Origin': '*',
20                "Access-Control-Allow-Credentials": True
21
22            },
23            "body": json.dumps({'answer': answer})
24        }
25    except Exception as e:
26        return {
27            "statusCode": 500,
28            "headers": {
29                'Content-Type': 'application/json',
30                'Access-Control-Allow-Origin': '*',
31                "Access-Control-Allow-Credentials": True
32            },
33            "body": json.dumps({"error": repr(e)})
34        }

Copy 
1{
2    "context": "We introduce a new language representation model called BERT, which stands for idirectional Encoder Representations from Transformers. Unlike recent language epresentation models (Peters et al., 2018a; Radford et al., 2018), BERT is designed to pretrain deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be finetuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial taskspecific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5% (7.7% point absolute improvement), MultiNLI accuracy to 86.7% (4.6% absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).",
3    "question": "What is BERTs best score on Squadv2 ?"
4}