14. Displaying the model’s activation activity

Last section we added the activation images to the return data of the /classify endpoint.

So, This time we'll display them.

Creating the activation visualisations takes a little while, so we're going to improve out app's responsiveness by adding a spinner.

We can grab a nice looking spinner component from the react-spinners library:

cd frontend
npm install --save react-spinners
cd ..

Create a new file at frontend/src/ImageGrid.tsx:

touch frontend/src/ImageGrid.tsx

The component is fairly simple:

interface ImageGridProps {
	activationImages: Record<string, string[]>;
}

function ImageGrid({ activationImages }: ImageGridProps) {
	return (
		<div
			style={{
				display: "grid",
				gridTemplateColumns: "repeat(4, 1fr) 2fr",
				gap: "16px",
			}}
		>
			{Object.keys(activationImages).map((layer) => (
				<div key={layer}>
					<h3>{layer}</h3>
					<div
						style={{
							display: "grid",
							gridTemplateColumns: `repeat(${
								activationImages[layer].length / 8
							}, 1fr)`,
							gap: "4px",
						}}
					>
						{activationImages[layer].map((img: string, idx: number) => (
							<img
								key={idx}
								src={`data:image/png;base64,${img}`}
								alt={`activation-${idx}`}
								width="100px"
								height="100px"
							/>
						))}
					</div>
				</div>
			))}
		</div>
	);
}

export default ImageGrid;

It takes an array of image strings as props, and displays them in a grid.

We then need to update the ImageUpload component.

Import the ImageGrid we just made, and your spinner of choice from react-spinners:

import { useCallback, useState } from "react";
import { useDropzone, FileWithPath } from "react-dropzone";
import GridLoader from "react-spinners/GridLoader";

import { Predictions } from "./Predictions";
import { ImageGrid } from "./ImageGrid";

Add state to store the activationImages, another called showActivationImages to toggle their display, and a third one to indicate whether the backend isLoading the request:

function ImageUpload() {
    const [imagePreview, setImagePreview] = useState<string | null>(null);
    const [predictedClass, setPredictedClass] = useState<string | null>(null);
    const [predictions, setPredictions] = useState<Record<string, number> | null>(null);
    const [activationImages, setActivationImages] = useState<any>({});
    const [showActivationImages, setShowActivationImages] = useState(false);
    const [isLoading, setIsLoading] = useState(false);

We'll need to reset these values when we run the onDrop callback:

// Reset state
setImagePreview(null);
setPredictedClass("");
setPredictions({});
setActivationImages({});
setShowActivationImages(false);
setIsLoading(false);

When we send the request, setIsLoading to true, then capture the activation_images we receive from the request, and finally setIsLoading back to false:

// Send the image to the backend for classification
setIsLoading(true);
fetch("/api/classify", {
	method: "POST",
	body: formData,
})
	.then((response) => response.json())
	.then((data) => {
		setPredictedClass(data["predicted_class"]);
		setPredictions(data["predictions"]);
		setActivationImages(data["activation_images"]);
	})
	.catch((error) => {
		console.log("Upload failed:", error);
	})
	.finally(() => {
		setIsLoading(false);
	});

We don't want the file explorer to open when we click the button (it will happen because the button is inside the drop element). So we need to define a customGetInputProps handler function, which will prevent the click event propergating:

const customGetInputProps = () => ({
	...getInputProps(),
	onClick: (event: React.MouseEvent<HTMLElement>) => {
		event.preventDefault();
		event.stopPropagation();
	},
});

If setShowActivationImages is true, then we want to return the ImageGrid to be rendered, and we'll add an onClick event handler to hide the grid again if we click it:

if (showActivationImages) {
	return (
		<div onClick={() => setShowActivationImages(false)}>
			<ImageGrid activationImages={activationImages} />
		</div>
	);
}

And then we adapt the render method to include a button, which will run setShowActivationImages(true):

    return (
        <div style={{ display: "flex", flexDirection: "column", alignItems: "center", backgroundColor: "#00000055", padding: "30px", borderRadius: "10px" }}>
            <div style={style} {...getRootProps()}>
                <input {...customGetInputProps()} />
                {imagePreview ? (
                    <div style={{ display: "flex", flexDirection: "column", alignItems: "center" }}>
                        <div style={{ display: "flex", flexDirection: "row", alignItems: "center" }}>
                            <div style={{ display: "flex", flexDirection: "column", alignItems: "center" }}>
                                <img src={imagePreview} width="400px" alt="Preview" />
                                {predictedClass ? <h1>{predictedClass}</h1> : <div className="spinner"></div>}
                            </div>
                            <Predictions predictions={predictions!} />
                        </div>
                        {Object.keys(activationImages).length > 0 ? <button onClick={() => setShowActivationImages(true)}>View Activations</button> : null}
                    </div>
                ) : (
                    <p>Drop an image here, or click to select</p>
                )}
                <GridLoader color={"teal"} loading={isLoading} size={25} aria-label="Loading Spinner" data-testid="loader" />
            </div>
        </div>
    );
}

We only want to display it when there are some images there to display, which we check for with the Object.keys(activationImages).length > 0 ternary operator.

We've also included the spinner towards the end of the HTML. This will hide and show itself depending on the value of isLoading.

When we build and run the server now, we should see a new button underneath the probabilities labelled: View Activations.

If we click that new button then we should see our activation images:

🏃 How to Run

🧱 Build Frontend

Navigate to the frontend/ directory:

cd frontend

Install any missing frontend dependancies:

npm install

Build the files for distributing the frontend to clients:

npm run build

🖲 Run the Backend

Go back to the project root directory:

cd ..

Activate the virtual environment, if you haven't already:

source .venv/bin/activate

Install any missing packages:

pip install -r requirements.txt

If you haven't already, train a CNN:

python scripts/train.py

Continue training an existing model:

python scripts/continue_training.py

Serve the web app:

python -m uvicorn main:app --port 8000 --reload

🚀 Deploy

Deploy to the cloud:

serverless deploy

Remove from the cloud:

severless remove

🗂️ Updated Files

Project structure

.
├── .venv/
├── .gitignore
├── .serverless/
├── resources
│   └── dog.jpg
├── frontend
│   ├── build/
│   ├── node_modules/
│   ├── public/
│   ├── src
│   │   ├── App.css
│   │   ├── App.test.tsx
│   │   ├── App.tsx
│   │   ├── ImageGrid.tsx
│   │   ├── ImageUpload.tsx
│   │   ├── index.css
│   │   ├── index.tsx
│   │   ├── Predictions.css
│   │   ├── Predictions.tsx
│   │   ├── logo.svg
│   │   ├── react-app-env.d.ts
│   │   ├── reportWebVitals.ts
│   │   ├── setupTests.ts
│   │   └── Sum.tsx
│   ├── .gitignore
│   ├── package-lock.json
│   ├── package.json
│   ├── README.md
│   └── tsconfig.json
├── output
│   ├── activations_conv2d/
│   ├── activations_conv2d_1/
│   ├── activations_conv2d_2/
│   ├── activations_dense/
│   ├── activations_dense_1/
│   ├── model.h5
│   ├── sample_images.png
│   └── training_history.png
├── scripts
│   ├── classify.py
│   ├── continue_training.py
│   └── train.py
├── main.py
├── README.md
├── requirements.txt
└── serverless.yml

frontend/src/ImageGrid.tsx

interface ImageGridProps {
	activationImages: Record<string, string[]>;
}

function ImageGrid({ activationImages }: ImageGridProps) {
	return (
		<div
			style={{
				display: "grid",
				gridTemplateColumns: "repeat(4, 1fr) 2fr",
				gap: "16px",
			}}
		>
			{Object.keys(activationImages).map((layer) => (
				<div key={layer}>
					<h3>{layer}</h3>
					<div
						style={{
							display: "grid",
							gridTemplateColumns: `repeat(${
								activationImages[layer].length / 8
							}, 1fr)`,
							gap: "4px",
						}}
					>
						{activationImages[layer].map((img: string, idx: number) => (
							<img
								key={idx}
								src={`data:image/png;base64,${img}`}
								alt={`activation-${idx}`}
								width="100px"
								height="100px"
							/>
						))}
					</div>
				</div>
			))}
		</div>
	);
}

export default ImageGrid;