diff --git a/.vscode/settings.json b/.vscode/settings.json index d14110b..1b456e2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -28,4 +28,7 @@ "editor.rulers": [80], }, "files.eol": "\n", + "cSpell.words": [ + "Airweave" + ], } diff --git a/main.py b/main.py index b98d6dd..9b209f0 100644 --- a/main.py +++ b/main.py @@ -3,31 +3,46 @@ This module is an Arweave FastAPI server that allows users to communicate with Arweave, and put Arkly files on chain. """ +import logging +import time from typing import Final, List -from fastapi import FastAPI, File, Form, Request, Response, UploadFile +from fastapi import FastAPI, File, Request, Response, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse from middleware import _update_db -from models import ArweaveTransaction +from models import Tags from primary_functions import ( + _all_transactions, _check_balance, - _check_balance_form, _check_last_transaction, _check_transaction_status, _create_transaction, - _create_transaction_form, _estimate_transaction_cost, + _fetch_tx_metadata, _fetch_upload, + _retrieve_by_tag_pair, _validate_bag, ) +logging.basicConfig( + format="%(asctime)-15s %(levelname)s :: %(filename)s:%(lineno)s:%(funcName)s() :: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level="INFO", +) + +logging.Formatter.converter = time.gmtime + +logger = logging.getLogger(__name__) + # Arkly-arweave API description. API_DESCRIPTION: Final[str] = " " # OpenAPI tags delineating the documentation. TAG_ARWEAVE: Final[str] = "arweave" +TAG_ARWEAVE_WALLET: Final[str] = "arweave wallet" +TAG_ARWEAVE_SEARCH: Final[str] = "arweave search" TAG_ARKLY: Final[str] = "arkly" # Metadata for each of the tags in the OpenAPI specification. To order @@ -37,6 +52,14 @@ "name": TAG_ARWEAVE, "description": "Manage Arweave transactions", }, + { + "name": TAG_ARWEAVE_WALLET, + "description": "Manage Arweave wallets", + }, + { + "name": TAG_ARWEAVE_SEARCH, + "description": "Search for Arweave transactions", + }, { "name": TAG_ARKLY, "description": "Arkly functions on-top of Arweave", @@ -46,7 +69,7 @@ app = FastAPI( title="api.arkly.io", description=API_DESCRIPTION, - version="2022.11.02.0001", + version="2023.08.09.0002", contact={ "": "", }, @@ -79,64 +102,93 @@ def redirect_root_to_docs(): return RedirectResponse(url="/docs") -@app.post("/check_balance/", tags=[TAG_ARWEAVE]) -async def check_balance(file: UploadFile = File(...)): +@app.post("/check_wallet_balance/", tags=[TAG_ARWEAVE_WALLET]) +async def check_wallet_balance(wallet: UploadFile): """Allows a user to check the balance of their wallet.""" - return await _check_balance(file) + return await _check_balance(wallet) -@app.post("/check_balance_form/", tags=[TAG_ARWEAVE]) -async def check_balance_form(wallet: str = Form()): - """Allows a user to check the balance of their wallet.""" - return await _check_balance_form(wallet) +@app.post("/check_wallet_last_transaction/", tags=[TAG_ARWEAVE_WALLET]) +async def check_wallet_last_transaction(wallet: UploadFile): + """Allows a user to check the transaction ID of their last + transaction. + """ + return await _check_last_transaction(wallet) -@app.post("/check_last_transaction/", tags=[TAG_ARWEAVE]) -async def check_last_transaction(file: UploadFile = File(...)): - """Allows a user to check the transaction id of their last - transaction. +@app.get("/estimate_transaction_cost/", tags=[TAG_ARWEAVE]) +async def estimate_transaction_cost(size_in_bytes: str): + """Allows a user to get an estimate of how much a transaction may + cost. """ - return await _check_last_transaction(file) + return await _estimate_transaction_cost(size_in_bytes) @app.get("/check_transaction_status/", tags=[TAG_ARWEAVE]) async def check_transaction_status(transaction_id: str): """Allows a user to check the transaction id of their last transaction. + + Example Tx: `rYa3ILXqWi_V52xPoG70y2EupPsTtu4MsMmz6DI4fy4` """ return await _check_transaction_status(transaction_id) -@app.get("/estimate_transaction_cost/", tags=[TAG_ARWEAVE]) -async def estimate_transaction_cost(size_in_bytes: str): - """Allows a user to get an estimate of how much a transaction may - cost. +@app.get("/fetch_transaction/", tags=[TAG_ARWEAVE]) +async def fetch_transaction(transaction_id: str): + """Allows a user to read their transaction files from the Arweave + blockchain. + + Example Tx: `rYa3ILXqWi_V52xPoG70y2EupPsTtu4MsMmz6DI4fy4` """ - return _estimate_transaction_cost(size_in_bytes) + return await _fetch_upload(transaction_id) -@app.get("/fetch_upload/", tags=[TAG_ARWEAVE]) -async def fetch_upload(transaction_id: str): - """Allows a user to read their file upload from the Arweave - blockchain. +@app.get("/fetch_transaction_metadata/", tags=[TAG_ARWEAVE]) +async def fetch_transaction_metadata(transaction_id: str): + """Fetch metadata from a given transaction ID to provide further + information about the uploaded package. + + Example Tx: `rYa3ILXqWi_V52xPoG70y2EupPsTtu4MsMmz6DI4fy4` """ - return await _fetch_upload(transaction_id) + return await _fetch_tx_metadata(transaction_id) -@app.post("/create_transaction/", tags=[TAG_ARKLY]) -async def create_transaction(files: List[UploadFile] = File(...)): - """Create an Arkly package and Arweave transaction.""" - return await _create_transaction(files) +@app.get("/all_wallet_transactions/", tags=[TAG_ARWEAVE_SEARCH]) +async def get_all_wallet_transactions(wallet_addr: str): + """Allows a user to see a list of all transactions with a given + wallet. + Example wallet: `6KymaAPWd3JNyMT0B7EPYij4TWxehhMrzRD8qifCSLs` + """ + return await _all_transactions(wallet_addr) + + +@app.get("/transactions_by_tag_pair/", tags=[TAG_ARWEAVE_SEARCH]) +async def get_transactions_by_tag_pair(name: str, value: str): + """Allows a user to retrieve transactions by tag-pair. + + Example tag key: `x-tag` + Example tag value: `arkly hello world!` + """ + return await _retrieve_by_tag_pair(name, value) -@app.post("/create_transaction_form/", tags=[TAG_ARKLY]) -async def create_transaction_form(transaction_json: ArweaveTransaction): + +@app.post("/create_transaction/", tags=[TAG_ARKLY]) +async def create_transaction( + wallet: UploadFile, + package_file_name: str, + files: List[UploadFile] = File(...), + tags: Tags | None = None, +): """Create an Arkly package and Arweave transaction.""" - data_files = await _create_transaction_form(transaction_json) - return await _create_transaction(data_files) + return await _create_transaction(wallet, files, package_file_name, tags) -@app.get("/validate_arweave_bag/", tags=[TAG_ARKLY]) +@app.get("/validate_arkly_bag/", tags=[TAG_ARKLY]) async def validate_bag(transaction_id: str, response: Response): - """Given an Arweave transaction ID, Validate an Arkly link as a bag.""" + """Given an Arweave transaction ID, Validate an Arkly link as a bag. + + Example Tx: `rYa3ILXqWi_V52xPoG70y2EupPsTtu4MsMmz6DI4fy4` + """ return await _validate_bag(transaction_id, response) diff --git a/middleware.py b/middleware.py index b28e5ec..e0ba46d 100644 --- a/middleware.py +++ b/middleware.py @@ -4,12 +4,14 @@ validation, and responses to be intercepted and augmented, e.g. adding headers, and other information. """ - +import logging from typing import Callable import psycopg2 from fastapi import Request +logger = logging.getLogger(__name__) + async def _update_db(request: Request, call_next: Callable): """Update the database by one per endpoint called.""" @@ -44,6 +46,6 @@ async def _update_db(request: Request, call_next: Callable): connection.commit() cursor.close() except psycopg2.DatabaseError as error: - print(error) + logger.warning("Postgres may not be configured correctly: %s", error) response = await call_next(request) return response diff --git a/models.py b/models.py index 2cfbe37..af8720e 100644 --- a/models.py +++ b/models.py @@ -1,4 +1,5 @@ """FastAPI models used in the Arweave API.""" +import json from typing import List from pydantic import BaseModel @@ -21,3 +22,75 @@ class ArweaveTransaction(BaseModel): ArweaveKey: str ArweaveFiles: List[FileItem] + + +class Tag(BaseModel): + """Describes the structure of a single tag for upload to Arweave. A + tag is simply a HTTP header and consists of a name and value. + + E.g. Name = `Content-type`, value = `application/gzip` + becomes `"Content-type: application/gzip"`. + """ + + name: str + value: str + + +# The Tags data type provides a way to provide an extensible list of +# data values, in this case, header tags for Airweave, where native +# HTTP handling makes this difficult as the form (from the FastAPI docs: +# is encoded as `application/x-www-form-urlencoded`. +# +# See also: +# +# * https://github.com/tiangolo/fastapi/issues/2257#issuecomment-727036089 +# * https://stackoverflow.com/a/70640522/21120938 +# * https://docs.pydantic.dev/1.10/usage/types/#classes-with-__get_validators__ +# +class Tags(BaseModel): + """Tags is an extensible data-type that allows users to provide + zero-to-many tags to supply to Airweave. + + To provide a value, provide a JSON object that looks something like + as follows: + + ```json + { + "tags": [ + { + "name": "tag_name_1", + "value": "tag_value_1" + }, + { + "name": "tag_name_2", + "value": "tag_value_2" + }, + { + "name": "tag_name_3", + "value": "tag_value_3" + } + ] + } + ``` + + """ + + # Default values are provided to help users understand how to use + # this data type. + # + # To create a Tag object you can do the following: + # Tag(**json.loads('{"name": "tag_name_1", "value": "tag_value_1"}')), + # + tags: list[Tag] = [] + + @classmethod + def __get_validators__(cls): + # pylint: disable=C0202 + yield cls.validate_to_json + + @classmethod + def validate_to_json(cls, value): + """Parse the input parameters and return a Tags instance.""" + if isinstance(value, str): + return cls(**json.loads(value)) + return value diff --git a/primary_functions.py b/primary_functions.py index 31938ad..7ee4740 100644 --- a/primary_functions.py +++ b/primary_functions.py @@ -3,11 +3,22 @@ These function calls are wrapped by the Arweave FastAPI endpoint calls. The FastAPI calls are used as entry points only to provide a place for formatted documentation. + +The current Arweave API client is found at: + + * https://github.com/MikeHibbert/arweave-python-client + +The Arweave API client is particularly helpful for working with Arweave +when the Arweave wallet is required. + +In some cases the Arweave API client is overridden with pure Arweave API +calls, especially "reading" where handling of wallets isn't required and +we can limit exposure of this information. """ import base64 import json +import logging import os -import sys import tarfile import tempfile from io import BytesIO @@ -16,15 +27,26 @@ import arweave import bagit +import humanize import requests -import ulid -from arweave.arweave_lib import Transaction +from arweave.arweave_lib import Transaction, arql from arweave.transaction_uploader import get_uploader -from fastapi import File, Form, HTTPException, Response, UploadFile, status +from fastapi import File, HTTPException, Response, UploadFile, status from fastapi.responses import FileResponse from arweave_utilities import winston_to_ar -from models import ArweaveTransaction +from models import Tags + +logger = logging.getLogger(__name__) + +ARWEAVE_API_BASEURL: Final[str] = "https://arweave.net" +ARWEAVE_VIEW_BASEURL: Final[str] = "https://arweave.app" + +ARKLY_AGENT = "arkly.io" + +# NB. Legacy code, we need to replace with different error handling. +# Beginning by incrementally working through the issues. +ERR_WALLET: Final[str] = "error handling wallet" def _file_from_data(file_data): @@ -43,15 +65,26 @@ async def create_temp_wallet(file: UploadFile) -> arweave.Wallet: :rtype: _type_ """ hold = await file.read() - json_obj = json.loads(hold) - wallet = arweave.Wallet.from_data(json_obj) + try: + json_obj = json.loads(hold) + except UnicodeDecodeError as err: + logger.error("wallet data is invalid, likely the wrong input format: %s", err) + return ERR_WALLET + try: + wallet = arweave.Wallet.from_data(json_obj) + except Exception as err: # pylint: disable=W0718 + # There are a range of Exceptions we need to try to catch here + # (I think), e.g. jose.exceptions.JWKError if the JSON is completely + # invalid but we need to bottom these out. Eventually we do not want + # to catch a bare-exception. + logger.error("error in Arweave Client API module: %s", err) if wallet is None: - print("Wallet object not made. Try another wallet, or try again.") - return "Error" + logger.error("wallet object not made. Try another wallet, or try again.") + return ERR_WALLET return wallet -async def _check_balance(file: UploadFile) -> dict: +async def _check_balance(wallet: UploadFile) -> dict: """Allows a user to check the balance of their wallet. :param file: JWK file, defaults to File(...) @@ -59,25 +92,15 @@ async def _check_balance(file: UploadFile) -> dict: :return: The balance of your wallet as a JSON object :rtype: JSON object """ - jwk_file = file + jwk_file = wallet wallet = await create_temp_wallet(jwk_file) - if wallet != "Error": + if wallet != ERR_WALLET: balance = wallet.balance return {"balance": balance} return {"balance": "Error on wallet load."} -async def _check_balance_form(wallet: str = Form()): - """Allows a user to check the balance of their wallet using a - application/x-www-form-urlencoded with wallet data encoded as a - Base64 string. - """ - bytes_wallet = _file_from_data(wallet) - uploaded_wallet = UploadFile(filename="", file=bytes_wallet, content_type="") - return await _check_balance(uploaded_wallet) - - -async def _check_last_transaction(file: UploadFile) -> dict: +async def _check_last_transaction(wallet: UploadFile) -> dict: """Allows a user to check the transaction id of their last transaction. @@ -86,12 +109,15 @@ async def _check_last_transaction(file: UploadFile) -> dict: :return: The transaction id as a JSON object :rtype: JSON object """ - wallet = await create_temp_wallet(file) - if wallet != "Error": + wallet = await create_temp_wallet(wallet) + if wallet != ERR_WALLET: last_transaction = requests.get( - f"https://arweave.net/wallet/{wallet.address}/last_tx" + f"{ARWEAVE_API_BASEURL}/wallet/{wallet.address}/last_tx" ) - return {"last_transaction_id": last_transaction.text} + return { + "wallet_address": f"{wallet.address}", + "last_transaction_id": f"{ARWEAVE_VIEW_BASEURL}/tx/{last_transaction.text}", + } return {"last_transaction_id": "Failure to get response..."} @@ -106,9 +132,9 @@ async def _check_transaction_status(transaction_id: int) -> dict: """ if len(transaction_id) == 43: transaction_status = requests.get( - f"https://arweave.net/tx/{transaction_id}/status" + f"{ARWEAVE_API_BASEURL}/tx/{transaction_id}/status" ) - return {"transaction_status": f"{transaction_status.text}"} + return {"transaction_status": json.loads(transaction_status.text)} return { "transaction_status": "Parameter issue. Please enter a valid transaction id." } @@ -117,6 +143,11 @@ async def _check_transaction_status(transaction_id: int) -> dict: async def _estimate_transaction_cost(size_in_bytes: str) -> dict: """Allows a user to get an estimate of how much a transaction may cost. + + Example cURL for this request: + + `curl -s https://arweave.net/price/1000/ | jq` + :param size_in_bytes: A string which is an integer that represents the number of bytes to be uploaded :type size_in_bytes: str @@ -124,14 +155,96 @@ async def _estimate_transaction_cost(size_in_bytes: str) -> dict: :rtype: JSON object """ if size_in_bytes.isdigit(): - cost_estimate = requests.get(f"https://arweave.net/price/{size_in_bytes}/") - winston_str = winston_to_ar(cost_estimate) + cost_estimate = requests.get(f"{ARWEAVE_API_BASEURL}/price/{size_in_bytes}/") + winston_str = winston_to_ar(cost_estimate.text) return {"estimate_transaction_cost": winston_str} return { "estimate_transaction_cost": "Parameter issue. Please enter a valid amount of bytes as an integer." } +def decode_base64_tag_fields(tags: list[str]) -> list[str]: + """Decode the tag fields from Base64 and return them in plain-text.""" + new_tags = [] + for item in tags: + new_item = {} + # new_name and new_value can potentially error with incorrect + # padding. We can fix this by adding the maximum amount of + # padding '==' and Python will truncate any excess. + # + # https://stackoverflow.com/a/49459036/21120938 + # + new_name = f"{item.get('name', '')}==".encode() + new_value = f"{item.get('value', '')}==".encode() + new_item["name"] = base64.b64decode(new_name) + new_item["value"] = base64.b64decode(new_value) + new_tags.append(new_item) + return new_tags + + +async def _fetch_tx_metadata(transaction_id: str) -> dict: + """Fetch tags for a transaction from Arweave given a transaction ID. + + The Arweave API client isn't used in this call as we don't want to + ask the user to supply their Wallet in the API request. Fetching Tx + details is purely a GET request. + + An example cURL request for the API call we mimic here is: + + * `curl -s https://arweave.net/tx/UQGNPIyhs2YFA569oSe-u-QPCn6q-w0IO9kGRnmq_Ak | jq` + + Details returned from the Tx metadata are: + + * "owner" + * "id" + * "tags" + + More information is available, described here: https://docs.arweave.org/developers/server/http-api#transaction-format + + For example: + + ``` + { + "format": 2, + "id": "BNttzDav3jHVnNiV7nYbQv-GY0HQ-4XXsdkE5K9ylHQ", + "last_tx": "jUcuEDZQy2fC6T3fHnGfYsw0D0Zl4NfuaXfwBOLiQtA", + "owner": "posmE...psEok", + "tags": [], + "target": "", + "quantity": "0", + "data_root": "PGh0b...RtbD4", + "data": "", + "data_size": "1234235", + "reward": "124145681682", + "signature": "HZRG_...jRGB-M" + } + ``` + + As the structure needs pre-processing to decode Base64 fields we simply + return a subset of this for ease of use. It can be expanded in future as + required. + """ + request_url = f"{ARWEAVE_API_BASEURL}/tx/{transaction_id}" + resp = requests.get(request_url, timeout=30) + data = json.loads(resp.text) + + # Humanize data size output for Arkly's end-users. + data["data_size_bytes"] = data["data_size"] + data["data_size_natural_size"] = humanize.naturalsize(data["data_size"]) + data.pop("data_size") + + # Decode Base64 enncoded tags returned from Arweave and replace + # in the return structure. + decoded_tags = decode_base64_tag_fields(data["tags"]) + data["tags"] = decoded_tags + + # Humanize reward output for Arkly's end-users. + data["reward_winston"] = data["reward"] + data["reward_ar"] = winston_to_ar(data["reward"]) + data.pop("reward") + return data + + async def _fetch_upload(transaction_id: str) -> FileResponse: """Allows a user to read their file upload from the Arweave blockchain. @@ -141,13 +254,13 @@ async def _fetch_upload(transaction_id: str) -> FileResponse: :return: The compressed file upload :rtype: File Object """ - url = "http://arweave.net/" + transaction_id + url = f"{ARWEAVE_API_BASEURL}/{transaction_id}" try: # Create a temporary directory for our fetch data. mkdtemp does # this in the most secure way possible. tmp_dir = tempfile.mkdtemp() fetch_dir = tmp_dir / Path(f"{transaction_id}.tar.gz") - print(f"Fetch writing to {fetch_dir}", file=sys.stderr) + logger.info("Fetch writing to %s", fetch_dir) response = requests.get(url) with open(str(fetch_dir), "wb") as content: content.write(response.content) @@ -156,37 +269,39 @@ async def _fetch_upload(transaction_id: str) -> FileResponse: raise HTTPException from err -async def bag_files(path: Path) -> None: +async def bag_files(path: Path, tag_list=None) -> None: """Use python Bagit to bag the files for Arkly-Arweave.""" - bagit.make_bag(path, {"Random Data": "arkly.io"}) - - -async def _package_content(files: List[UploadFile] = File(...)) -> dict: + if not tag_list: + bagit.make_bag(path, {"packaging-agent": ARKLY_AGENT}) + return + + bag_info = {} + for tag in tag_list: + bag_info[f"{tag.name}".replace(" ", "-")] = tag.value + bag_info["packaging-agent"] = ARKLY_AGENT + logger.info("writing package with bag-info: %d", bag_info) + bagit.make_bag(path, bag_info) + return + + +async def _package_content( + files: List[UploadFile] = File(...), package_name: str = None, tag_list: list = None +) -> dict: """Package the files submitted to the create_transaction endpoint.""" # Create a folder for the user's wallet. tmp_dir = tempfile.mkdtemp() - package_ulid = str(ulid.new()) - file_path = Path(tmp_dir, package_ulid) + file_path = Path(tmp_dir, package_name) file_path.mkdir() - print("Location to write object to:", file_path, file=sys.stderr) + logger.info("Location to write object to: %s", file_path) for file in files: read_file = await file.read() output_file = Path(file_path, file.filename) output_file.write_bytes(read_file) - # Create a metadata path for the bag. - metadata_path = file_path / Path(f".{package_ulid}") - metadata_path.mkdir(parents=True) - metadata = metadata_path / Path(package_ulid).with_suffix(".md") - - # Write some mock metadata for demo. Move this to a separate built - # for purpose function later. - metadata.write_text('{"dc:creator": "api.arkly.io"}') - # Bag these files. - await bag_files(file_path) + await bag_files(file_path, tag_list) # Create compressed .tar.gz file tar_file_name = file_path.with_suffix(".tar.gz") @@ -200,7 +315,12 @@ async def _package_content(files: List[UploadFile] = File(...)) -> dict: return tar_file_name -async def _create_transaction(files: List[UploadFile] = File(...)) -> dict: +async def _create_transaction( + wallet: UploadFile, + files: List[UploadFile] = File(...), + package_file_name: str = None, + tags: Tags = None, +) -> dict: """Create an Arkly package and Arweave transaction. We do so as follows: @@ -211,66 +331,56 @@ async def _create_transaction(files: List[UploadFile] = File(...)) -> dict: - Uploads the compressed tarball to Arweave for the current Arweave price. """ - for file in files: - wallet = await create_temp_wallet(file) - if wallet != "Error": - files.remove(file) - break - if wallet != "Error": - # Create a package from files array. Package content will create - # this in a secure temporary directory. - tar_file_name = await _package_content(files) - - print("Adding version to package:", tar_file_name, file=sys.stderr) - print("New path exists:", tar_file_name.is_file(), file=sys.stderr) - print("Wallet balance before upload:", wallet.balance, file=sys.stderr) - - print(wallet.balance) - - with open(tar_file_name, "rb", buffering=0) as file_handler: - new_transaction = Transaction( - wallet, file_handler=file_handler, file_path=tar_file_name - ) - new_transaction.add_tag("Content-Type", "application/gzip") - new_transaction.sign() - uploader = get_uploader(new_transaction, file_handler) - while not uploader.is_complete: - uploader.upload_chunk() - - print("Finished!") - tx_status = new_transaction.get_status() - print(tx_status, file=sys.stderr) - print(new_transaction.id, file=sys.stderr) - print(wallet.balance, file=sys.stderr) - return { - "transaction_id": f"{new_transaction.id}", - "transaction_link": f"https://viewblock.io/arweave/tx/{new_transaction.id}", - "transaction_status": f"{tx_status}", - "wallet_balance": f"{wallet.balance}", - } - return {"transaction_id": "Error creating transaction."} - - -async def _create_transaction_form(transaction_json: ArweaveTransaction): - """Create an Arkly package and Arweave transaction using an - application/x-www-form-urlencoded form, with data encoded as - Base64 strings. - """ - arweave_file_item_list = transaction_json.ArweaveFiles - bytes_wallet = _file_from_data(transaction_json.ArweaveKey) - data_files = [ - UploadFile(filename="wallet.json", file=bytes_wallet, content_type="text/json"), - ] - # Iterate through FileItem objects. - for file_item in arweave_file_item_list: - print(file_item) - print(type(file_item)) - bytes_packet = _file_from_data(file_item.Base64File) - upload_obj = UploadFile( - filename=file_item.FileName, file=bytes_packet, content_type="text/plain" + wallet = await create_temp_wallet(wallet) + if wallet == ERR_WALLET: + return {"transaction_id": "Error creating transaction."} + if wallet.balance <= 0: + return {"transaction_id": f"Error: wallet balance is: {wallet.balance}"} + if not files: + return {"transaction_id": "Error: no files selected for upload"} + + tag_list = [] + try: + # Process user-defined tags. + tag_list = tags.tags + except AttributeError: + logger.info("no user-defined tags provided by caller") + + # Create a package from files array. Package content will create + # this in a secure temporary directory. + tar_file_name = await _package_content(files, package_file_name, tag_list) + + logger.info("Adding version to package: %s", tar_file_name) + logger.info("New path exists: %s", tar_file_name.is_file()) + logger.info("Wallet balance before upload: %s", wallet.balance) + + with open(tar_file_name, "rb", buffering=0) as file_handler: + new_transaction = Transaction( + wallet, file_handler=file_handler, file_path=tar_file_name ) - data_files.append(upload_obj) - return data_files + # Default tags for the tar/gzip file that we create. + new_transaction.add_tag("Content-Type", "application/gzip") + + for tag in tag_list: + logger.info("Adding tag: %s: %s", tag.name, tag.value) + new_transaction.add_tag(tag.name, tag.value) + + new_transaction.sign() + uploader = get_uploader(new_transaction, file_handler) + while not uploader.is_complete: + uploader.upload_chunk() + + logger.info("Finished uploading to Arkly!") + tx_status = new_transaction.get_status() + logger.info("Transaction status: %s", tx_status) + logger.info("Transaction ID: %s", new_transaction.id) + logger.info("New wallet balance: %s", wallet.balance) + return { + "transaction_id": f"{new_transaction.id}", + "transaction_link": f"{ARWEAVE_VIEW_BASEURL}/tx/{new_transaction.id}", + "transaction_status": f"{tx_status}", + "wallet_balance": f"{wallet.balance}", + } def _get_arweave_urls_from_tx(transaction_id: str) -> dict: @@ -278,8 +388,8 @@ def _get_arweave_urls_from_tx(transaction_id: str) -> dict: transaction ID. """ return ( - f"https://viewblock.io/arweave/tx/{transaction_id}", - f"https://arweave.net/{transaction_id}", + f"{ARWEAVE_VIEW_BASEURL}/tx/{transaction_id}", + f"{ARWEAVE_API_BASEURL}/{transaction_id}", ) @@ -308,8 +418,8 @@ async def _validate_bag(transaction_id: str, response: Response) -> dict: } try: - bag_ulid = os.listdir(tmp_dir)[0] - bag_file = Path(tmp_dir) / bag_ulid + bag_name = os.listdir(tmp_dir)[0] + bag_file = Path(tmp_dir) / bag_name except IndexError: response.status_code = status.HTTP_404_NOT_FOUND return { @@ -326,7 +436,7 @@ async def _validate_bag(transaction_id: str, response: Response) -> dict: "file_url": arweave_url, "valid": f"{arkly_bag.validate()}", "bag_info": arkly_bag.info, - "bag_ulid": bag_ulid, + "bag_name": bag_name, } except bagit.BagError: response.status_code = status.HTTP_422_UNPROCESSABLE_ENTITY @@ -335,3 +445,39 @@ async def _validate_bag(transaction_id: str, response: Response) -> dict: "file_url": arweave_url, "valid": "UNKNOWN", } + + +async def _all_transactions(wallet_addr: str): + """Retrieve all transactions from a given wallet and return a human + friendly link to enable users to view the transaction. + """ + query = {"op": "equals", "expr1": "from", "expr2": f"{wallet_addr}"} + tx_ids = arql(None, query) + if not tx_ids: + return { + "wallet_address": f"{wallet_addr}", + "total_transactions": len(tx_ids), + "arweave_transactions": tx_ids, + } + tx_uris = [f"{ARWEAVE_VIEW_BASEURL}/tx/{tx}" for tx in tx_ids] + return { + "wallet_address": f"{wallet_addr}", + "total_transactions": len(tx_uris), + "arweave_transactions": tx_uris, + } + + +async def _retrieve_by_tag_pair(name: str, value: str) -> dict: + """Retrieve all transactions with a given tag-pair.""" + query = {"op": "equals", "expr1": f"{name}", "expr2": f"{value}"} + tx_ids = arql(None, query) + if not tx_ids: + return { + "total_transactions": 0, + "arweave_transactions": [], + } + tx_uris = [f"{ARWEAVE_VIEW_BASEURL}/tx/{tx}" for tx in tx_ids] + return { + "tag_pair": {"name": f"{name}", "value": f"{value}"}, + "arweave_transactions": tx_uris, + } diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 1d6eb91..55c80dd 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,10 +1,11 @@ # Requirements for arkly-io. +arweave-python-client==1.0.19 bagit==1.8.1 -fastapi==0.79.0 -uvicorn==0.18.2 -arweave-python-client==1.0.18 +fastapi==0.99.1 +humanize==4.7.0 +psycopg2-binary==2.9.3 python-multipart==0.0.5 requests==2.28.1 ulid-py==1.1.0 -psycopg2-binary==2.9.3 +uvicorn==0.22.0