This document provides an example of how to ingest custom logs into Multicloud Observability Platform. The scenario involves ingesting CloudFlare logs into the Multicloud Observability Platform platform via Huawei Object Block Store (OBS).
Example Python script:
import sys import hashlib import hmac import binascii from datetime import datetime import json import requests def generate_signature(method, secret_key, date, bucket, key): canonical_string = f"{method}\n\n\n{date}\n/{bucket}/{key}" hashed = hmac.new(secret_key.encode('UTF-8'), canonical_string.encode('UTF-8'), hashlib.sha1) return binascii.b2a_base64(hashed.digest()).strip().decode('UTF-8') def handler(event, context): bucket_name = event['Records'][0]['s3']['bucket']['name'] object_key = event['Records'][0]['s3']['object']['key'] secret_access_key = context.getUserData('API_SECRET') access_key_id = context.getUserData('API_ACCESS_KEY') date = datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT') signature = generate_signature("GET", secret_access_key, date, bucket_name, object_key) url = f"https://{bucket_name}.obs.ap-southeast-3.myhuaweicloud.com/{object_key}" headers = {'Authorization': f'OBS {access_key_id}:{signature}', 'Date': date} response = requests.get(url, headers=headers) response.raise_for_status() response_text_modified = response.text.replace("}\n{", "},\n{") json_array = json.loads(f'[{response_text_modified}]') datakit_url = "http://<YOUR_DATAKIT_IP>:9529/v1/write/logstreaming?type=firelens&source=cf_hw_obs" for item in json_array: requests.post(datakit_url, json=item) return {"statusCode": 200, "body": json.dumps(event)}
Example pipeline script:
jsonObj = get_key(message) if valid_json(jsonObj): jsonObj = load_json(jsonObj) add_key("CacheCacheStatus", jsonObj["CacheCacheStatus"]) add_key("ClientCountry", jsonObj["ClientCountry"]) add_key("ClientIP", jsonObj["ClientIP"]) add_key("EdgeResponseBytes", jsonObj["EdgeResponseBytes"]) add_key("EdgeResponseStatus", jsonObj["EdgeResponseStatus"]) add_key("EdgeStartTimestamp", jsonObj["EdgeStartTimestamp"]) add_key("RayID", jsonObj["RayID"])