minio-aws-使用

发表于 | in . | | 访客

使用minio调用s3接口，来操作其他对象存储

安装

1
# pip install boto3  ## 这里使用python，boto3是aws的python api实现

启用minio服务

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# wget https://dl.minio.io/server/minio/release/linux-amd64/minio
# chmod +x minio
# 每个业务对应一个monio server,启动多个server, 通过 :9001 /data/test2 这个两个来生成key和端口及数据目录
# ./minio server --address :9000 /data/test1   ## 把其输出保存，以备他用, 后面创建的bucket和文件都存储在/data/test1目录下
Endpoint:  http://172.20.0.151:9000  http://127.0.0.1:9000
AccessKey: G6R3JD4NSA072AGRYTQL
SecretKey: Jp+z9bSfxOEZ+hW8mTJrr+K9+sb3k4RAr+c55gxR
Browser Access:
   http://172.20.0.151:9000  http://127.0.0.1:9000
Command-line Access: https://docs.minio.io/docs/minio-client-quickstart-guide
   $ mc config host add myminio http://172.20.0.151:9000 G6R3JD4NSA072AGRYTQL Jp+z9bSfxOEZ+hW8mTJrr+K9+sb3k4RAr+c55gxR
Object API (Amazon S3 compatible):
   Go:         https://docs.minio.io/docs/golang-client-quickstart-guide
   Java:       https://docs.minio.io/docs/java-client-quickstart-guide
   Python:     https://docs.minio.io/docs/python-client-quickstart-guide
   JavaScript: https://docs.minio.io/docs/javascript-client-quickstart-guide
   .NET:       https://docs.minio.io/docs/dotnet-client-quickstart-guide

web 访问 minio

http://172.20.0.151:9000
输入上面产生的key
然后在web上创建一个bucket：test ,会对应在目录/data/test1 下生成目录 test
如果上传文件，则在/data/test1/test 目录下存储该文件

调用s3接口使用minio

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
cat s3.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import boto3
import json
import os
import sys
import threading
# 前面通过 ./minion server --address :9000 /data/test1 生成的minio 的key
ACCESS_KEY_ID="G6R3JD4NSA072AGRYTQL"
ACCESS_KEY="Jp+z9bSfxOEZ+hW8mTJrr+K9+sb3k4RAr+c55gxR"
BUCKET_NAME="test"
UNIT = {"KB": 1024, "MB": 1024 * 1024, "GB": 1024 * 1024 * 1024, "TB": 1024 * 1024 * 1024 * 1024, "PB": 1024 * 1024 * 1024 * 1024 * 1024}
class ProgressPercentage(object):
        def __init__(self, filename):
            self._filename = filename
            self._size = float(os.path.getsize(filename))
            self._seen_so_far = 0
            self._lock = threading.Lock()
        def __call__(self, bytes_amount):
            with self._lock:
                self._seen_so_far += bytes_amount
                percentage = (self._seen_so_far / self._size) * 100
                sys.stdout.write("\r%s / %s  (%.2f%%)" % ( self._seen_so_far, self._size, percentage))
                sys.stdout.flush()
class AWS(object):
    """
      初始化客户端
    """
    def __init__(self):
        self.s3client = boto3.client('s3', aws_access_key_id=ACCESS_KEY_ID,
                                      aws_secret_access_key=ACCESS_KEY,
                                      endpoint_url='http://172.20.0.151:9000',  # minion 的访问url
                                      region_name='cn-north-1')
        self.config = boto3.s3.transfer.TransferConfig(multipart_threshold=24 * 1024 * 1024,
                                                            max_concurrency=20, multipart_chunksize=24 * 1024 * 1024)
    """
      上传文件
      file_path: 待上传文件路径
      save_name: 保存到s3上的文件名,下载时会用到该名称
    """
    def upload_file(self, file_path, save_name):
        self.s3client.upload_file(file_path, BUCKET_NAME, save_name, Callback=ProgressPercentage(file_path), Config=self.config)
    """
        删除文件
    """
    def delete_file(self, name):
        self.s3client.delete_object(Bucket=BUCKET_NAME, Key=name)
    """
      查看s3文件列表
      bucket_name: 存储桶名称
    """
    def list_objects(self, bucket_name):
        #bucket = self.s3resource.Bucket(bucket_name)
        #result = bucket.objects.all()
        result = self.s3client.list_objects_v2(Bucket=bucket_name)
        objects = []
        total_size = 0
        objs_contents = result['Contents'] if result.has_key('Contents') else []
        for i in range(len(objs_contents)):
            total_size = total_size + objs_contents[i]['Size']
            objects.append({"LastModified": objs_contents[i]['LastModified'].strftime('%Y-%m-%d %H:%M:%S'),
                            "Size": objs_contents[i]['Size'], "Key": objs_contents[i]['Key']})
        return objects, total_size
    """
      下载文件
      bucket_name: 存储桶名称
      obj_key: 保存到s3上的文件对象名称
      save_name: 保存到本地的文件名称
    """
    def download_file(self, bucket_name, obj_key, save_name):
        return self.s3client.download_file(bucket_name, obj_key, save_name)
    """
      统计存储量
      bucket_name：存储桶名称
    """
    def used_size(self, kucket_name, unit):
        objects, total_size = self.list_objects(kucket_name)
        return "%.2f" % (total_size / float(UNIT[unit])) + unit
if __name__=="__main__":
    aws = AWS()
    ##文件上传
    aws.upload_file('helloworld.txt', 'helloworld.txt')
    #aws.upload_file('CentOS7.2-1511-Base-v1.4.img', 'CentOS7.2-1511-Base-v1.5.img')
    ##文件下载
    #aws.download_file(BUCKET_NAME, 'helloworld.txt', 'helloworld1.txt')
    ## 文件删除
    #aws.delete_file('CentOS7.2-1511-Base-v1.5.img')
    ##查看s3上的文件
    print aws.list_objects(BUCKET_NAME)
    # 查看使用率
    #print aws.used_size(BUCKET_NAME, "GB")
# python s3.py