S3 Sharding Examples

The following examples show different ways to implement S3 sharding.

S3 Sharding Example 1

In the following sample configuration, the filestore is implemented with two S3 shards, one region, and one redundancy.

<config version="2">

 <chain>
        <provider id="cache-fs" type="cache-fs">
            <provider type="sharding" id="sharding">
                <redundancy>2</redundancy>
                <sub-provider type="state-aware-s3" id="s3-shard1"/>
                <sub-provider type="state-aware-s3" id="s3-shard2"/>
            </provider>
        </provider>
 </chain>

 <provider id="sharding" type="sharding">
      <redundancy>1</redundancy>
 </provider>
   
 <provider id="s3-shard1" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>data1212</bucketName>
      <path>yon1220d</path>
      <region>us-east-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
 </provider>
 <provider id="s3-shard2" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>test-tomers-bucket</bucketName>
      <path>yon1220t</path>
      <region>us-east-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
 </provider>
</config>

S3 Sharding Example 2

In the following sample configuration, the filestore is implemented with two S3 shards, one region, and two redundancy.

<config version="2">
 <chain> <!-- template="s3-sharding" -->
        <provider id="cache-fs" type="cache-fs">
            <provider type="sharding" id="sharding">
                <redundancy>2</redundancy>
                <sub-provider type="state-aware-s3" id="s3-shard1"/>
                <sub-provider type="state-aware-s3" id="s3-shard2"/>
            </provider>
        </provider>
    </chain>   

   <provider id="sharding" type="sharding">
      <redundancy>2</redundancy>
   </provider>
   <provider id="s3-shard1" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>bucket1</bucketName>
      <path>path1</path>
      <region>us-east-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
   </provider>
   <provider id="s3-shard2" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>bucket2</bucketName>
      <path>path2</path>
      <region>us-east-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
   </provider>
</config>

S3 Sharding Example 3

In the following sample configuration, the filestore is implemented with five S3 shards, two region, and two redundancy.

<?xml version="1.0" encoding="UTF-8"?>
<config version="5">
 <chain> <!-- template="s3-sharding" -->
        <provider id="cache-fs" type="cache-fs">
            <provider type="sharding" id="sharding">
                <redundancy>2</redundancy>
                <sub-provider type="state-aware-s3" id="s3-shard1" />
                <sub-provider type="state-aware-s3" id="s3-shard2" />
                <sub-provider type="state-aware-s3" id="s3-shard3" />
                <sub-provider type="state-aware-s3" id="s3-shard4" />
                <sub-provider type="state-aware-s3" id="s3-shard5" />
            </provider>
        </provider>
    </chain>

   <provider id="sharding" type="sharding">
      <redundancy>3</redundancy>
   </provider>
    <provider id="s3-shard1" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>data120</bucketName>
      <path>yond</path>
      <region>us-east-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
   </provider>
   <provider id="s3-shard2" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>data125</bucketName>
      <path>yont</path>
      <region>us-east-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <writeEnabled>false</writeEnabled>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
   </provider>
   <provider id="s3-shard3" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>data121</bucketName>
      <path>yonb</path>
      <region>us-west-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
   </provider>
   <provider id="s3-shard4" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>data122</bucketName>
      <path>yonb2</path>
      <region>us-east-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
   </provider>
   <provider id="s3-shard5" type="state-aware-s3">
      <endpoint>http://s3.amazonaws.com</endpoint>
      <bucketName>data123</bucketName>
      <path>yonb3</path>
      <region>us-west-1</region>
      <provider.id>aws-s3</provider.id>
      <identity>AK...-accessKeyId</identity>
      <credential>ePE...-secretAccessKey</credential>
      <enableSignedUrlRedirect>true</enableSignedUrlRedirect>
      <signedUrlExpirySeconds>3600</signedUrlExpirySeconds>
      <testConnection>false</testConnection>
      <max.retry.number>2</max.retry.number>
   </provider>
</config>