@ -38,6 +38,7 @@ import (
"sort"
"sort"
"strconv"
"strconv"
"strings"
"strings"
"sync"
"time"
"time"
"github.com/dustin/go-humanize"
"github.com/dustin/go-humanize"
@ -1054,6 +1055,153 @@ func testGetObjectWithVersioning() {
successLogger ( testName , function , args , startTime ) . Info ( )
successLogger ( testName , function , args , startTime ) . Info ( )
}
}
func testPutObjectWithVersioning ( ) {
// initialize logging params
startTime := time . Now ( )
testName := getFuncName ( )
function := "GetObject()"
args := map [ string ] interface { } { }
// Seed random based on current time.
rand . Seed ( time . Now ( ) . Unix ( ) )
// Instantiate new minio client object.
c , err := minio . New ( os . Getenv ( serverEndpoint ) ,
& minio . Options {
Creds : credentials . NewStaticV4 ( os . Getenv ( accessKey ) , os . Getenv ( secretKey ) , "" ) ,
Secure : mustParseBool ( os . Getenv ( enableHTTPS ) ) ,
} )
if err != nil {
logError ( testName , function , args , startTime , "" , "MinIO client object creation failed" , err )
return
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c . SetAppInfo ( "MinIO-go-FunctionalTest" , "0.1.0" )
// Generate a new random bucket name.
bucketName := randString ( 60 , rand . NewSource ( time . Now ( ) . UnixNano ( ) ) , "minio-go-test-" )
args [ "bucketName" ] = bucketName
// Make a new bucket.
err = c . MakeBucket ( context . Background ( ) , bucketName , minio . MakeBucketOptions { Region : "us-east-1" , ObjectLocking : true } )
if err != nil {
logError ( testName , function , args , startTime , "" , "Make bucket failed" , err )
return
}
err = c . EnableVersioning ( context . Background ( ) , bucketName )
if err != nil {
logError ( testName , function , args , startTime , "" , "Enable versioning failed" , err )
return
}
objectName := randString ( 60 , rand . NewSource ( time . Now ( ) . UnixNano ( ) ) , "" )
args [ "objectName" ] = objectName
const n = 10
// Read input...
// Save the data concurrently.
var wg sync . WaitGroup
wg . Add ( n )
var buffers = make ( [ ] [ ] byte , n )
var errs [ n ] error
for i := 0 ; i < n ; i ++ {
r := newRandomReader ( int64 ( ( 1 << 20 ) * i + i ) , int64 ( i ) )
buf , err := ioutil . ReadAll ( r )
if err != nil {
logError ( testName , function , args , startTime , "" , "unexpected failure" , err )
return
}
buffers [ i ] = buf
go func ( i int ) {
defer wg . Done ( )
_ , errs [ i ] = c . PutObject ( context . Background ( ) , bucketName , objectName , bytes . NewReader ( buf ) , int64 ( len ( buf ) ) , minio . PutObjectOptions { PartSize : 5 << 20 } )
} ( i )
}
wg . Wait ( )
for _ , err := range errs {
if err != nil {
logError ( testName , function , args , startTime , "" , "PutObject failed" , err )
return
}
}
objectsInfo := c . ListObjects ( context . Background ( ) , bucketName , minio . ListObjectsOptions { WithVersions : true , Recursive : true } )
var results [ ] minio . ObjectInfo
for info := range objectsInfo {
if info . Err != nil {
logError ( testName , function , args , startTime , "" , "Unexpected error during listing objects" , err )
return
}
results = append ( results , info )
}
if len ( results ) != n {
logError ( testName , function , args , startTime , "" , "Unexpected number of Version elements in listing objects" , nil )
return
}
sort . Slice ( results , func ( i , j int ) bool {
return results [ i ] . Size < results [ j ] . Size
} )
sort . Slice ( buffers , func ( i , j int ) bool {
return len ( buffers [ i ] ) < len ( buffers [ j ] )
} )
for i := 0 ; i < len ( results ) ; i ++ {
opts := minio . GetObjectOptions { VersionID : results [ i ] . VersionID }
reader , err := c . GetObject ( context . Background ( ) , bucketName , objectName , opts )
if err != nil {
logError ( testName , function , args , startTime , "" , "error during GET object" , err )
return
}
statInfo , err := reader . Stat ( )
if err != nil {
logError ( testName , function , args , startTime , "" , "error during calling reader.Stat()" , err )
return
}
if statInfo . ETag != results [ i ] . ETag {
logError ( testName , function , args , startTime , "" , "error during HEAD object, unexpected ETag" , err )
return
}
if statInfo . LastModified . Unix ( ) != results [ i ] . LastModified . Unix ( ) {
logError ( testName , function , args , startTime , "" , "error during HEAD object, unexpected Last-Modified" , err )
return
}
if statInfo . Size != results [ i ] . Size {
logError ( testName , function , args , startTime , "" , "error during HEAD object, unexpected Content-Length" , err )
return
}
tmpBuffer := bytes . NewBuffer ( [ ] byte { } )
_ , err = io . Copy ( tmpBuffer , reader )
if err != nil {
logError ( testName , function , args , startTime , "" , "unexpected io.Copy()" , err )
return
}
if ! bytes . Equal ( tmpBuffer . Bytes ( ) , buffers [ i ] ) {
logError ( testName , function , args , startTime , "" , "unexpected content of GetObject()" , err )
return
}
}
// Delete all objects and their versions as long as the bucket itself
if err = cleanupVersionedBucket ( bucketName , c ) ; err != nil {
logError ( testName , function , args , startTime , "" , "CleanupBucket failed" , err )
return
}
successLogger ( testName , function , args , startTime ) . Info ( )
}
func testCopyObjectWithVersioning ( ) {
func testCopyObjectWithVersioning ( ) {
// initialize logging params
// initialize logging params
startTime := time . Now ( )
startTime := time . Now ( )
@ -1191,6 +1339,166 @@ func testCopyObjectWithVersioning() {
successLogger ( testName , function , args , startTime ) . Info ( )
successLogger ( testName , function , args , startTime ) . Info ( )
}
}
func testConcurrentCopyObjectWithVersioning ( ) {
// initialize logging params
startTime := time . Now ( )
testName := getFuncName ( )
function := "CopyObject()"
args := map [ string ] interface { } { }
// Seed random based on current time.
rand . Seed ( time . Now ( ) . Unix ( ) )
// Instantiate new minio client object.
c , err := minio . New ( os . Getenv ( serverEndpoint ) ,
& minio . Options {
Creds : credentials . NewStaticV4 ( os . Getenv ( accessKey ) , os . Getenv ( secretKey ) , "" ) ,
Secure : mustParseBool ( os . Getenv ( enableHTTPS ) ) ,
} )
if err != nil {
logError ( testName , function , args , startTime , "" , "MinIO client object creation failed" , err )
return
}
// Enable tracing, write to stderr.
// c.TraceOn(os.Stderr)
// Set user agent.
c . SetAppInfo ( "MinIO-go-FunctionalTest" , "0.1.0" )
// Generate a new random bucket name.
bucketName := randString ( 60 , rand . NewSource ( time . Now ( ) . UnixNano ( ) ) , "minio-go-test-" )
args [ "bucketName" ] = bucketName
// Make a new bucket.
err = c . MakeBucket ( context . Background ( ) , bucketName , minio . MakeBucketOptions { Region : "us-east-1" , ObjectLocking : true } )
if err != nil {
logError ( testName , function , args , startTime , "" , "Make bucket failed" , err )
return
}
err = c . EnableVersioning ( context . Background ( ) , bucketName )
if err != nil {
logError ( testName , function , args , startTime , "" , "Enable versioning failed" , err )
return
}
objectName := randString ( 60 , rand . NewSource ( time . Now ( ) . UnixNano ( ) ) , "" )
args [ "objectName" ] = objectName
var testFiles = [ ] string { "datafile-10-kB" }
for _ , testFile := range testFiles {
r := getDataReader ( testFile )
buf , err := ioutil . ReadAll ( r )
if err != nil {
logError ( testName , function , args , startTime , "" , "unexpected failure" , err )
return
}
r . Close ( )
_ , err = c . PutObject ( context . Background ( ) , bucketName , objectName , bytes . NewReader ( buf ) , int64 ( len ( buf ) ) , minio . PutObjectOptions { } )
if err != nil {
logError ( testName , function , args , startTime , "" , "PutObject failed" , err )
return
}
}
objectsInfo := c . ListObjects ( context . Background ( ) , bucketName , minio . ListObjectsOptions { WithVersions : true , Recursive : true } )
var infos [ ] minio . ObjectInfo
for info := range objectsInfo {
if info . Err != nil {
logError ( testName , function , args , startTime , "" , "Unexpected error during listing objects" , err )
return
}
infos = append ( infos , info )
}
sort . Slice ( infos , func ( i , j int ) bool {
return infos [ i ] . Size < infos [ j ] . Size
} )
reader , err := c . GetObject ( context . Background ( ) , bucketName , objectName , minio . GetObjectOptions { VersionID : infos [ 0 ] . VersionID } )
if err != nil {
logError ( testName , function , args , startTime , "" , "GetObject of the oldest version content failed" , err )
return
}
oldestContent , err := ioutil . ReadAll ( reader )
if err != nil {
logError ( testName , function , args , startTime , "" , "Reading the oldest object version failed" , err )
return
}
// Copy Source
srcOpts := minio . CopySrcOptions {
Bucket : bucketName ,
Object : objectName ,
VersionID : infos [ 0 ] . VersionID ,
}
args [ "src" ] = srcOpts
dstOpts := minio . CopyDestOptions {
Bucket : bucketName ,
Object : objectName + "-copy" ,
}
args [ "dst" ] = dstOpts
// Perform the Copy concurrently
const n = 10
var wg sync . WaitGroup
wg . Add ( n )
var errs [ n ] error
for i := 0 ; i < n ; i ++ {
go func ( i int ) {
defer wg . Done ( )
_ , errs [ i ] = c . CopyObject ( context . Background ( ) , dstOpts , srcOpts )
} ( i )
}
wg . Wait ( )
for _ , err := range errs {
if err != nil {
logError ( testName , function , args , startTime , "" , "CopyObject failed" , err )
return
}
}
objectsInfo = c . ListObjects ( context . Background ( ) , bucketName , minio . ListObjectsOptions { WithVersions : true , Recursive : false , Prefix : dstOpts . Object } )
infos = [ ] minio . ObjectInfo { }
for info := range objectsInfo {
// Destination object
readerCopy , err := c . GetObject ( context . Background ( ) , bucketName , objectName + "-copy" , minio . GetObjectOptions { VersionID : info . VersionID } )
if err != nil {
logError ( testName , function , args , startTime , "" , "GetObject failed" , err )
return
}
defer readerCopy . Close ( )
newestContent , err := ioutil . ReadAll ( readerCopy )
if err != nil {
logError ( testName , function , args , startTime , "" , "Reading from GetObject reader failed" , err )
return
}
if len ( newestContent ) == 0 || ! bytes . Equal ( oldestContent , newestContent ) {
logError ( testName , function , args , startTime , "" , "Unexpected destination object content" , err )
return
}
infos = append ( infos , info )
}
if len ( infos ) != n {
logError ( testName , function , args , startTime , "" , "Unexpected number of Version elements in listing objects" , nil )
return
}
// Delete all objects and their versions as long as the bucket itself
if err = cleanupVersionedBucket ( bucketName , c ) ; err != nil {
logError ( testName , function , args , startTime , "" , "CleanupBucket failed" , err )
return
}
successLogger ( testName , function , args , startTime ) . Info ( )
}
func testComposeObjectWithVersioning ( ) {
func testComposeObjectWithVersioning ( ) {
// initialize logging params
// initialize logging params
startTime := time . Now ( )
startTime := time . Now ( )
@ -7548,7 +7856,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() {
completeParts = append ( completeParts , minio . CompletePart { PartNumber : part . PartNumber , ETag : part . ETag } )
completeParts = append ( completeParts , minio . CompletePart { PartNumber : part . PartNumber , ETag : part . ETag } )
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , bucketName , objectName , uploadID , completeParts )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , bucketName , objectName , uploadID , completeParts , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -7606,7 +7914,7 @@ func testSSECMultipartEncryptedToSSECCopyObjectPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -7783,7 +8091,7 @@ func testSSECEncryptedToSSECCopyObjectPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -7959,7 +8267,7 @@ func testSSECEncryptedToUnencryptedCopyPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -8138,7 +8446,7 @@ func testSSECEncryptedToSSES3CopyObjectPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -8312,7 +8620,7 @@ func testUnencryptedToSSECCopyObjectPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -8482,7 +8790,7 @@ func testUnencryptedToUnencryptedCopyPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -8654,7 +8962,7 @@ func testUnencryptedToSSES3CopyObjectPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -8829,7 +9137,7 @@ func testSSES3EncryptedToSSECCopyObjectPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -9000,7 +9308,7 @@ func testSSES3EncryptedToUnencryptedCopyPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -9174,7 +9482,7 @@ func testSSES3EncryptedToSSES3CopyObjectPart() {
}
}
// Complete the multipart upload
// Complete the multipart upload
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } )
_ , err = c . CompleteMultipartUpload ( context . Background ( ) , destBucketName , destObjectName , uploadID , [ ] minio . CompletePart { fstPart , sndPart , lstPart } , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
logError ( testName , function , args , startTime , "" , "CompleteMultipartUpload call failed" , err )
return
return
@ -11285,22 +11593,20 @@ func testRemoveObjects() {
var reader = getDataReader ( "datafile-129-MB" )
var reader = getDataReader ( "datafile-129-MB" )
defer reader . Close ( )
defer reader . Close ( )
n, err : = c . PutObject ( context . Background ( ) , bucketName , objectName , reader , int64 ( bufSize ) , minio . PutObjectOptions { } )
_, err = c . PutObject ( context . Background ( ) , bucketName , objectName , reader , int64 ( bufSize ) , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
log . Fatalln ( err )
logError ( testName , function , args , startTime , "" , "Error uploading object" , err )
}
}
log . Println ( "Uploaded" , objectName , " of size: " , n , "to bucket: " , bucketName , "Successfully." )
// Replace with smaller...
// Replace with smaller...
bufSize = dataFileMap [ "datafile-10-kB" ]
bufSize = dataFileMap [ "datafile-10-kB" ]
reader = getDataReader ( "datafile-10-kB" )
reader = getDataReader ( "datafile-10-kB" )
defer reader . Close ( )
defer reader . Close ( )
n , err = c . PutObject ( context . Background ( ) , bucketName , objectName , reader , int64 ( bufSize ) , minio . PutObjectOptions { } )
_ , err = c . PutObject ( context . Background ( ) , bucketName , objectName , reader , int64 ( bufSize ) , minio . PutObjectOptions { } )
if err != nil {
if err != nil {
log . Fatalln ( err )
logError ( testName , function , args , startTime , "" , "Error uploading object" , err )
}
}
log . Println ( "Uploaded" , objectName , " of size: " , n , "to bucket: " , bucketName , "Successfully." )
t := time . Date ( 2030 , time . April , 25 , 14 , 0 , 0 , 0 , time . UTC )
t := time . Date ( 2030 , time . April , 25 , 14 , 0 , 0 , 0 , time . UTC )
m := minio . RetentionMode ( minio . Governance )
m := minio . RetentionMode ( minio . Governance )
@ -11416,6 +11722,7 @@ func main() {
testFPutObjectContextV2 ( )
testFPutObjectContextV2 ( )
testFGetObjectContextV2 ( )
testFGetObjectContextV2 ( )
testPutObjectContextV2 ( )
testPutObjectContextV2 ( )
testPutObjectWithVersioning ( )
testMakeBucketError ( )
testMakeBucketError ( )
testMakeBucketRegions ( )
testMakeBucketRegions ( )
testPutObjectWithMetadata ( )
testPutObjectWithMetadata ( )
@ -11453,6 +11760,7 @@ func main() {
testStatObjectWithVersioning ( )
testStatObjectWithVersioning ( )
testGetObjectWithVersioning ( )
testGetObjectWithVersioning ( )
testCopyObjectWithVersioning ( )
testCopyObjectWithVersioning ( )
testConcurrentCopyObjectWithVersioning ( )
testComposeObjectWithVersioning ( )
testComposeObjectWithVersioning ( )
testRemoveObjectWithVersioning ( )
testRemoveObjectWithVersioning ( )
testRemoveObjectsWithVersioning ( )
testRemoveObjectsWithVersioning ( )