Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use size and mtime to determine if a file has changed rather than hash #460

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Use size and mtime to determine if a file has changed rather than hash #460

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
809e69c
Use size and mtime to determine if a file has changed rather than has…
FraserThompson Aug 28, 2023
c27cf95
Move stuff back into the parallel queue and handle new files
FraserThompson Aug 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 61 additions & 43 deletions gatsby-plugin-s3/src/bin.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import minimatch from 'minimatch';
import mime from 'mime';
import inquirer from 'inquirer';
import { config as awsConfig } from 'aws-sdk';
import { createHash } from 'crypto';
import isCI from 'is-ci';
import { getS3WebsiteDomainUrl, withoutLeadingSlash } from './util';
import { AsyncFunction, asyncify, parallelLimit } from 'async';
Expand Down Expand Up @@ -106,6 +105,24 @@ const createSafeS3Key = (key: string): string => {
return key;
};

const isFileUnchanged = (localStats: fs.Stats, s3Object?: S3.Object) => {

// If there's no S3 object it's a new file, so we'll say it's changed
if (!s3Object) return false;

const remoteModifiedTime = s3Object.LastModified;

if (remoteModifiedTime){
const localModifiedTime = localStats.mtime;
// AWS doesn't have millisecond precision
remoteModifiedTime?.setMilliseconds(0);
localModifiedTime.setMilliseconds(0);
return s3Object.Size === localStats.size && remoteModifiedTime.getTime() >= localModifiedTime.getTime();
} else {
return s3Object.Size === localStats.size;
}
}

export interface DeployArguments {
yes?: boolean;
bucket?: string;
Expand All @@ -114,6 +131,7 @@ export interface DeployArguments {
export const deploy = async ({ yes, bucket, userAgent }: DeployArguments = {}) => {
const spinner = ora({ text: 'Retrieving bucket info...', color: 'magenta', stream: process.stdout }).start();
let dontPrompt = yes;
let uploadCount = 0;

const uploadQueue: Array<AsyncFunction<void, Error>> = [];

Expand Down Expand Up @@ -233,9 +251,9 @@ export const deploy = async ({ yes, bucket, userAgent }: DeployArguments = {}) =
spinner.text = 'Listing objects...';
spinner.color = 'green';
const objects = await listAllObjects(s3, config.bucketName, config.bucketPrefix);
const keyToETagMap = objects.reduce((acc: { [key: string]: string }, curr) => {
if (curr.Key && curr.ETag) {
acc[curr.Key] = curr.ETag;
const keyToObjectMap = objects.reduce((acc: { [key: string]: S3.Object}, curr) => {
if (curr.Key) {
acc[curr.Key] = curr;
}
return acc;
}, {});
Expand All @@ -247,58 +265,59 @@ export const deploy = async ({ yes, bucket, userAgent }: DeployArguments = {}) =
const isKeyInUse: { [objectKey: string]: boolean } = {};

stream.on('data', ({ path, stats }) => {
if (!stats.isFile()) {
return;
}
uploadQueue.push(
asyncify(async () => {
if (!stats.isFile()) {
return;
}

let key = createSafeS3Key(relative(publicDir, path));
if (config.bucketPrefix) {
key = `${config.bucketPrefix}/${key}`;
}
const readStream = fs.createReadStream(path);
const hashStream = readStream.pipe(createHash('md5').setEncoding('hex'));
const data = await streamToPromise(hashStream);
isKeyInUse[key] = true;

const tag = `"${data}"`;
const objectUnchanged = keyToETagMap[key] === tag;
const objectUnchanged = isFileUnchanged(stats, keyToObjectMap[key]);

isKeyInUse[key] = true;
if (objectUnchanged) {
spinner.text = chalk`Syncing...\n{dim Skipping {cyan ${key}}} (unchanged)`;
return;
}

if (!objectUnchanged) {
try {
const upload = new S3.ManagedUpload({
service: s3,
params: {
Bucket: config.bucketName,
Key: key,
Body: fs.createReadStream(path),
ACL: config.acl === null ? undefined : config.acl ?? 'public-read',
ContentType: mime.getType(path) ?? 'application/octet-stream',
...getParams(key, params),
},
});

upload.on('httpUploadProgress', evt => {
spinner.text = chalk`Syncing...
try {
const upload = new S3.ManagedUpload({
service: s3,
params: {
Bucket: config.bucketName,
Key: key,
Body: fs.createReadStream(path),
ACL: config.acl === null ? undefined : config.acl ?? 'public-read',
ContentType: mime.getType(path) ?? 'application/octet-stream',
...getParams(key, params),
},
});

upload.on('httpUploadProgress', evt => {
spinner.text = chalk`Syncing...
{dim Uploading {cyan ${key}} ${evt.loaded.toString()}/${evt.total.toString()}}`;
});
});

await upload.promise();
spinner.text = chalk`Syncing...\n{dim Uploaded {cyan ${key}}}`;
} catch (ex) {
console.error(ex);
process.exit(1);
}
await upload.promise();
uploadCount++;
spinner.text = chalk`Syncing...\n{dim Uploaded {cyan ${key}}}`;
} catch (ex) {
console.error(ex);
process.exit(1);
}
})
);
});

const base = config.protocol && config.hostname ? `${config.protocol}://${config.hostname}` : null;
redirectObjects.forEach(redirect =>
redirectObjects.forEach(redirect => {
uploadQueue.push(
asyncify(async () => {

const { fromPath, toPath: redirectPath } = redirect;
const redirectLocation = base ? resolveUrl(base, redirectPath) : redirectPath;

Expand All @@ -311,15 +330,12 @@ export const deploy = async ({ yes, bucket, userAgent }: DeployArguments = {}) =
key = withoutLeadingSlash(`${config.bucketPrefix}/${key}`);
}

const tag = `"${createHash('md5')
.update(redirectLocation)
.digest('hex')}"`;
const objectUnchanged = keyToETagMap[key] === tag;
const stats = fs.statSync(fromPath)
const objectUnchanged = isFileUnchanged(stats, keyToObjectMap[key]);

isKeyInUse[key] = true;

if (objectUnchanged) {
// object with exact hash already exists, abort.
return;
}

Expand Down Expand Up @@ -348,7 +364,7 @@ export const deploy = async ({ yes, bucket, userAgent }: DeployArguments = {}) =
}
})
)
);
});

await streamToPromise(stream as Readable);
await promisifiedParallelLimit(uploadQueue, config.parallelLimit as number);
Expand Down Expand Up @@ -391,11 +407,13 @@ export const deploy = async ({ yes, bucket, userAgent }: DeployArguments = {}) =
console.log(chalk`
{bold Your website is online at:}
{blue.underline http://${config.bucketName}.${s3WebsiteDomain}}
{dim ${uploadCount.toString()} files were uploaded.}
`);
} else {
console.log(chalk`
{bold Your website has now been published to:}
{blue.underline ${config.bucketName}}
{dim ${uploadCount.toString()} files were uploaded.}
`);
}
} catch (ex) {
Expand Down