From fc6131e3aee819f5912fdaf2d0fec460291050bd Mon Sep 17 00:00:00 2001 From: Ammar Hussein Date: Tue, 16 Sep 2025 20:43:56 -0700 Subject: [PATCH] Added minio support --- .gitignore | 2 ++ README.md | 14 +++++++++++++- lib/.empty | 1 + lib/main.py | 25 +++++++++++++++++++++++++ lib/minio_helper.py | 30 ++++++++++++++++++++++++++++++ requirements.txt | 9 +++++++++ 6 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 lib/.empty create mode 100644 lib/main.py create mode 100644 lib/minio_helper.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index b9bb42d..09833da 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ geckodriver-* tmp/ *.org~ data/ +venv/ +*~ \ No newline at end of file diff --git a/README.md b/README.md index 88926d4..3ed9f13 100644 --- a/README.md +++ b/README.md @@ -22,4 +22,16 @@ Relevant but not 1-1 walkthrough of how to programmatically find building owners [eRealProperty](https://kingcounty.gov/en/dept/kcit/data-information-services/gis-center/property-research): King County assessor data for finding the owner of a given parcel. [Washington State Corporations and Charities Filing Database (CCFS)](https://kingcounty.gov/en/dept/kcit/data-information-services/gis-center/property-research): For looking up a parcel owner name and finding the related business listing and related info. -TODO: Find a good source for eviction filing data. Those with access can refer to the [potential data source list](https://docs.google.com/spreadsheets/d/1Ew0UrZvP-S74velkWSKaiSGBYcxIAoRH6IGpEzNWX6s/edit?gid=0#gid=0) to find new data sources. \ No newline at end of file +TODO: Find a good source for eviction filing data. Those with access can refer to the [potential data source list](https://docs.google.com/spreadsheets/d/1Ew0UrZvP-S74velkWSKaiSGBYcxIAoRH6IGpEzNWX6s/edit?gid=0#gid=0) to find new data sources. + +## Object Storage + +An S3 compatible storage is hosted on [minio.radmin.live](minio.radmin.live) + +SDK documentation: https://github.com/minio/minio-py/blob/master/docs/API.md + +Use `lib/minio_helper.py` to extend the functionality + +Run `test_minio` in `lib/main.py` to test out that it works (TODO: move this to own testing script, perhaps unit tests) + +Note: You will need to have access_key and secret_key in your env before running for this to work, contact @linnealovespie or @ammaratef45 to obtain these keys) diff --git a/lib/.empty b/lib/.empty new file mode 100644 index 0000000..31bc1ec --- /dev/null +++ b/lib/.empty @@ -0,0 +1 @@ +Well, not really empty, but it is for testing, please keep as is! \ No newline at end of file diff --git a/lib/main.py b/lib/main.py new file mode 100644 index 0000000..a83e7fa --- /dev/null +++ b/lib/main.py @@ -0,0 +1,25 @@ +from minio_helper import MinioHelper + +def test_minio(): + minio_helper = MinioHelper('evictorbook-seattle-data') + files_list = minio_helper.list_files() + print('listing current files stored in the bucket') + for file in files_list: + print(file) + print('adding a file then listing again') + minio_helper.upload('.empty') + files_list = minio_helper.list_files() + for file in files_list: + print(file) + print('deleting the file then listing again') + minio_helper.delete('.empty') + files_list = minio_helper.list_files() + for file in files_list: + print(file) + + +def main(): + test_minio() + +if __name__ == '__main__': + main() diff --git a/lib/minio_helper.py b/lib/minio_helper.py new file mode 100644 index 0000000..ba6392b --- /dev/null +++ b/lib/minio_helper.py @@ -0,0 +1,30 @@ +from minio import Minio +import io +import os + +class MinioHelper: + + def __init__(self, bucket_name: str): + self.client = Minio( + "minio.radmin.live", + access_key=os.environ['access_key'], + secret_key=os.environ['secret_key'] + ) + self.bucket_name = bucket_name + + def list_files(self): + return self.client.list_objects(self.bucket_name) + + def upload(self, filepath: str): + with open(filepath, "rb") as fh: + buf = io.BytesIO(fh.read()) + self.client.put_object( + self.bucket_name, + filepath, + buf, + length=-1, + part_size=10*1024*1024 + ) + + def delete(self, filepath: str): + self.client.remove_object(self.bucket_name, filepath) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5794235 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +argon2-cffi==25.1.0 +argon2-cffi-bindings==25.1.0 +certifi==2025.8.3 +cffi==2.0.0 +minio==7.2.16 +pycparser==2.23 +pycryptodome==3.23.0 +typing_extensions==4.15.0 +urllib3==2.5.0 -- 2.49.0