Cognitive Services Tutorial #4: Create a photo-upload app

In this exercise, you will create a new Web app in Visual Studio Code and add code to upload images, write them to blob storage, display them in a Web page, generate thumbnails, captions, and keywords using the Computer Vision API, and perform keyword searches on uploaded images. The app will be named Intellipix (for “Intelligent Pictures”) and will be accessed through your browser. The server-side code will be written in JavaScript and Node.js. The code that runs in the browser will be written in JavaScript and will leverage two of the most popular class libraries on the planet: AngularJS and Bootstrap

1.Open a Command Prompt window if you’re using Windows, or a Terminal window if you’re using macOS or Linux. Then execute the following command, substituting the Computer Vision API key you copied to the clipboard in the previous exercise for vision_api_key:
 Windows:

set AZURE_VISION_API_KEY=vision_api_key

Mac or Linux:

export AZURE_VISION_API_KEY=vision_api_key

2.Next, execute the following command, substituting the name of the storage account you created in Exercise 1 for storage_account_name:
Windows:

set AZURE_STORAGE_ACCOUNT=storage_account_name

Mac or Linux:

export AZURE_STORAGE_ACCOUNT=storage_account_name

3.Now type the following command, replacing storage_account_key with the access key that you saved in Tutorial 1, Step 9:
Windows:

set AZURE_STORAGE_ACCESS_KEY=storage_account_key

Mac or Linux:

export AZURE_STORAGE_ACCESS_KEY=storage_account_key

4.Create a project directory named “Intellipix” in the location of your choice. In the Command Prompt or Terminal window, navigate to that directory and execute the following command (note the space and the period at the end of the command) to start Visual Studio Code in that directory:

 code .

5.In Visual Studio Code, click the Source Control button in the activity bar on the left.

6.Click Initialize Repository to initialize a Git repository in the working directory and place the directory under source control.

Return to the Command Prompt or Terminal window and make sure you’re still in the “Intellipix” directory that you created for the project (the directory that was just placed under source control). Then execute the following command to initialize the project. If prompted for an author name, enter your name.

npm init -y

8.Now execute the following command to install the NPM packages that the app will use:

npm install -save azure-storage express multer request streamifier

9.Return to Visual Studio Code and click the Explorer button in the upper-left corner. Then click package.json to open that file for editing.

10.Add the following statements to package.json just before the “keywords” definition. Then save your changes.

"engines": {
  "node": ">=4.0"
},

11.  Place the mouse cursor over “INTELLIPIX” in the Explorer window and click the New File button to add a file to the project root. Name the new file .gitigonore. Be sure to include the leading period in the file name.

12. Add the following statements to .gtignore to exclude the specified directions from source control:

.vscode/
node_modules/

13. Add a file named server.js to the root of the project and insert the following statements:

  1. var express = require('express');
    var multer = require('multer');
    var azureStorage = require('azure-storage');
    var streamifier = require('streamifier');
    var request = require('request');
    
    var portNum = process.env.PORT || 9898;
    var endpoint = 'vision_api_endpoint';
    
    var app = express();
    var storage = multer.memoryStorage();
    var uploadImage = multer({ storage: storage }).single('imageFile');
    
    app.post('/api/image-upload', configurationMiddleware, uploadImage, imageHandlerMiddleware);
    app.get('/api/images', configurationMiddleware, noCacheMiddleware, listBlobsMiddleware);
    app.use('/', express.static('src'));
    app.use(errorHandlerMiddleware);
    
    app.listen(portNum, function() {
        console.log("Web application listening on port " + portNum);
    });
    
    function configurationMiddleware(req, res, next) {
    
        var verifyConfigValue = function(keyName) {
            var configValue = process.env[keyName];
            if(!configValue) {
                throw new Error(keyName + " not defined.");
            }
            return configValue;
        };
    
        req.appConfig = {
            storageAccount: verifyConfigValue("AZURE_STORAGE_ACCOUNT"),
            storageAccountAccessKey: verifyConfigValue("AZURE_STORAGE_ACCESS_KEY"),
            visionApiKey: verifyConfigValue("AZURE_VISION_API_KEY")
        };
        next();
    }
    
    function imageHandlerMiddleware(req, res) {
    
        // Note, all of this work is done in memory!!
        var cfg = req.appConfig;
        var uploadFile = req.file;
        var blobService = azureStorage.createBlobService(cfg.storageAccount, cfg.storageAccountAccessKey);
        var publicUrl = [
            "https://",
            cfg.storageAccount,
            ".blob.core.windows.net/photos/",
            uploadFile.originalname
        ].join('');
    
        console.log(["Received ", uploadFile.originalname, " (", uploadFile.size, " bytes)"].join(''));
        saveImageToAzure(uploadFile);
    
        function saveImageToAzure() {
            blobService.createBlockBlobFromStream(
                'photos',
                uploadFile.originalname,
                streamifier.createReadStream(uploadFile.buffer),
                uploadFile.size,
                function(err, result, response) {
                    if(err){
                        throw err;
                    }
                    console.log(["Uploaded ", uploadFile.originalname, " image to 'photos' container on Azure."].join(''));
                    console.log(["URL: ", publicUrl].join(''));
                    createThumbnailOfImage();
                });
        }
    
    function createThumbnailOfImage(){
        var options = {
            url: endpoint + "/generateThumbnail",
            qs: {
                width: 192,
                height: 128,
                smartCropping: true
            },
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Ocp-Apim-Subscription-Key': cfg.visionApiKey
            },
            json: true,
            body: {
                url: publicUrl
            }
        };
        request(options)
            .on('error', function(err) {
                throw err;
            })
            .on('end', function() {
                console.log(["Created ", uploadFile.originalname, " thumbnail."].join(''));
            })
            .pipe(saveThumbnailToAzure());
        }
    
        function saveThumbnailToAzure() {
            return blobService
                .createWriteStreamToBlockBlob('thumbnails', uploadFile.originalname)
                .on('error', function(err) {
                    throw err;
                })
                .on('end', function() {
                    console.log(["Uploaded ", uploadFile.originalname, " image to 'thumbnails' container on Azure."].join(''));
                    analyzeImage();
                });
        }
    
        function analyzeImage() {
            var options = {
                url: endpoint + "/analyze",
                qs: {
                    visualFeatures: "Description"
                },
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                    'Ocp-Apim-Subscription-Key': cfg.visionApiKey
                },
                json: true,
                body: {
                    url: publicUrl
                }
            };
            request(options, function(err, response, body) {
                if(err) {
                    throw err;
                }
                console.log(["Analyzed ", uploadFile.originalname].join(''));
                saveAnalysisResults(body);
            });
        }
    
        function saveAnalysisResults(result) {
            var metaData = {
                caption: result.description && result.description.captions && result.description.captions.length ?
                    result.description.captions[0].text :
                    "Unknown",
                tags: result.description && result.description.tags && result.description.tags.length ?
                    JSON.stringify(result.description.tags) :
                    []
            };
    
            blobService.setBlobMetadata(
                'photos',
                uploadFile.originalname,
                metaData,
                function(err, result, response) {
                    if(err){
                        throw err;
                    }
                    console.log(["Stored ", uploadFile.originalname, " analysis results to Azure."].join(''));
                    res.status(200).send({
                        name: uploadFile.originalname,
                        mimetype: uploadFile.mimetype,
                        result: result
                    });
                });
        }
    }
    
    function listBlobsMiddleware(req, res) {
        var cfg = req.appConfig;
        var blobService = azureStorage.createBlobService(cfg.storageAccount, cfg.storageAccountAccessKey);
        var options = {
            maxResults: 5000,
            include: "metadata",
    
        };
        blobService.listBlobsSegmented(
            'photos',
            null,
            options,
            function(err, result, response) {
                if(err) {
                    throw err;
                }
                (result.entries || []).forEach(function(entry) {
                    entry.url = [
                        "https://",
                        cfg.storageAccount,
                        ".blob.core.windows.net/thumbnails/",
                        entry.name
                    ].join("");
                    entry.fullUrl = [
                        "https://",
                        cfg.storageAccount,
                        ".blob.core.windows.net/photos/",
                        entry.name
                    ].join("");
                    entry.metadata = entry.metadata || {};
                });
                res.status(200).json(result);
            }
        )
    }
    
    function noCacheMiddleware(req, res, next) {
        res.header('Cache-Control', 'private, no-cache, no-store, must-revalidate');
        res.header('Expires', '-1');
        res.header('Pragma', 'no-cache');
        next();
    }
    
    function errorHandlerMiddleware(err, req, res, next) {
        console.error(err);
        res.status(500).send({
            error: true,
            message: err.toString()
        });
    }

    This is the code that executes in Node.js on the server. Points of interest include the saveImageToAzure function, which saves an uploaded image in blob storage using APIs in the Azure Storage Client Library for Node.js, the createThumbnailFromImage function, which uses the Computer Vision API to generate an image thumbnail, and the analyzeImage function, which uses the Computer Vision API to generate a caption and a list of keywords describing the image. Another function you might care to inspect is saveAnalysisResults, which writes the caption and keywords to blob metadata. Finally, take a moment to examine the listBlobsMiddleware function, which enumerates the photos uploaded to the site by enumerating the blobs in the “photos” container.

  2. Replace vision_api_endpoint on line 8 with the Computer Vision API endpoint that you saved in Exercise 3, Step 4. Add “vision/v1.0” to the end of the URL if it isn’t already there.

  3. Place the mouse cursor over “INTELLIPIX” in Visual Studio Code’s Explorer window and click the New Folder button that appears. Name the new folder “src” (without quotation marks).

    Adding a folder

    Adding a folder

  4. Right-click (on a Mac, Command-click) the “src” folder and use the New File command to add a file named index.html to the “src” folder. Then insert the following statements into index.html:

    <!DOCTYPE html>
    <html lang="en" ng-app="myApp">
    <head>
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <title>Intellipix</title>
    <link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet">
    <style>
        .ng-cloak { display: none !important; }
        a img {
        	cursor: pointer;
        }
        	.image-modal img {
        width: 100%;
        }
    </style>
    </head>
    <body class="ng-cloak">
    
    <div class="container body-content" ng-controller="mainCtrl as ctrl">
    
        <h1>Intellipix</h1>
    
        <!-- Panel containing image-upload and search controls -->
        <div class="well">
        <form ng-if="!ctrl.analysis.inProgress">
            <div class="row">
            <div class="col-md-7">
                <div class="form-group">
                <label for="imageFile">Select Image to Analyze:</label>
                <input type="file" id="imageFile" name="imageFile" on-file-selected="ctrl.imageFileSelected(file)">
                </div>
            </div>
            <div class="col-md-5">
                <div class="form-group">
                <label for="searchText">Search:</label>
                <div class="input-group">
                    <input type="text" id="searchText" ng-model="ctrl.searchText" class="form-control">
                    <span class="input-group-btn">
                    <button class="btn btn-default" type="button" ng-click="ctrl.clearSearchText()">
                        <span class="glyphicon glyphicon-remove" aria-hidden="true"></span>
                    </button>
                    </span>
                </div>
                </div>
            </div>
            </div>
        </form>
        <p ng-if="ctrl.analysis.inProgress">
            <span class="glyphicon glyphicon-time" aria-hidden="true"></span>
            Analyzing Image ...
        </p>
        </div>
    
        <!-- Panel showing error message if upload or image analysis fails -->
        <div class="alert alert-danger alert-dismissible" role="alert" ng-if="ctrl.analysis.error">
        <button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">&times;</span></button>
        <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
        {{ctrl.analysis.error.message}}
        </div>
    
        <!-- Thumbnail images -->
        <div class="row">
        <div class="col-sm-12">
            <a ng-click="ctrl.showImageDetails(img)" ng-repeat="img in ctrl.images | filter:ctrl.imageFilter">
                <img ng-src="{{img.url}}" width="192" ng-attr-title={{img.metadata.caption}} style="padding-right: 16px; padding-bottom: 16px">
            </a>
        </div>
        </div>
    
        <!-- Modal window used to show enlarged images -->
        <div class="modal fade image-modal" id="imageModal" tabindex="-1" role="dialog" aria-labelledby="myModalLabel">
        <div class="modal-dialog modal-lg" role="document">
            <div class="modal-content">
            <div class="modal-header">
                <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
                <h4 class="modal-title" id="myModalLabel">{{ctrl.current.metadata.caption}}</h4>
            </div>
            <div class="modal-body">
                <img ng-src="{{ctrl.current.fullUrl}}">
            </div>
            </div>
        </div>
        </div>
    </div>
    
    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.6/js/bootstrap.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.6/angular.min.js"></script>
    <script src="index.js"></script>
    
    </body>
    </html>

    This is the HTML file containing the site’s one and only page. It uses Bootstrap’s grid layout system to align elements on the page, and it uses AngularJS to make the page dynamic. Notice the ng- attributes such as ng-click and ng-src attached to some of the page’s elements, as well as the “mustache” expressions in double curly braces (for example, {{ctrl.current.metadata.caption}}). These attributes and expressions are part of AngularJS and are frequently found in pages that use it.

  5. Add a file named index.js to the “src” folder and insert the following statements:

    (function() {
    
        function mainController($http) {
            this.$http = $http;
            this.analysis = {
                inProgress: false
            };
            this.images = [];
            this.current = null;
            this.searchText = '';
            this.imageFilter = imageFilter.bind(this);
            this.loadImageList();
    
            function imageFilter(img) {
                var search = this.searchText;
                var tags = img && img.metadata && img.metadata.tags;
    
                if(!search || !tags) {
                    return true;
                }
    
                if(containsText(tags, search)) {
                    return true;
                }
    
                return false;
            }
        }
        mainController.prototype = {
    
            clearSearchText: function() {
                this.searchText = '';
            },
    
            imageFileSelected: function(file) {
                var ctrl = this, formData;
                ctrl.analysis = {
                    inProgress: true
                };
                formData = new FormData();
                formData.append('imageFile', file);
                ctrl.$http
                    .post('/api/image-upload', formData, {
                        transformRequest: angular.identity,
                        headers: {
                            'Content-Type': undefined
                        }
                    })
                    .then(function(result) {
                        ctrl.analysis = {
                            inProgress: false,
                            result: result.data
                        };
                        ctrl.loadImageList();
                    })
                    .catch(function(err) {
                        ctrl.analysis = {
                            inProgress: false,
                            error: err.data || { message: err.statusText }
                        };
                    });
            },
    
            loadImageList: function() {
                var ctrl = this;
                ctrl.$http.get('/api/images')
                    .then(function(result) {
                        ctrl.images = result.data.entries || [];
                    })
                    .catch(function(err) {
                        alert((err.data && err.data.message) || err.toString());
                    });
            },
    
            showImageDetails: function(img) {
                this.current = img;
                angular.element("#imageModal").modal();
            }
        };
    
        function fileContentBinderDirective() {
            return {
                restrict: 'A',
                scope: {
                    onFileSelected: '&'
                },
                link: function(scope, element) {
                    element.on('change', function() {
                        scope.$apply(function() {
                            scope.onFileSelected({
                                file: element[0].files[0]
                            });
                        });
                    });
                }
            };
        }
    
        function containsText(text, search) {
            return (text && search) ? (text.toLowerCase().indexOf(search.toLowerCase()) > -1) : false;
        }
    
        angular
            .module('myApp', [])
            .controller('mainCtrl', ['$http', mainController])
            .directive('onFileSelected', [fileContentBinderDirective]);
    
    }());

This is the code that executes in Node.js on the server. Points of interest include the saveImageToAzure function, which saves an uploaded image in blob storage using APIs in the Azure Storage Client Library for Node.js, the createThumbailFromImage function, which uses Computer Vision API to generate an image thumbnail, and the analyzeImage function, which use the Computer Vision API to generate a caption and a list of keywords to blob metadata. Finally, take a moment to examine the listBlobsMiddleware function, which enumerates the photo uploaded to the site by enumerating the blobs in the “photo” container.

14. Replace vision_api_endpoint on line 8 with the Computer Vision endpoint that you saved in previous Tutorial. Add “vision/v1.0” to the end of the URL if isn’t already there.

15. Place the mouse cursor “INTELLIPIX” in Visual Studio Code’s Explorer window and click the New Folder button that appears, Name the new folder “src” (withount quotation marks).

16. Right-click(on a Mac, Command-click) the “src” folder and use the New File command to add a file named index.html to the “src” folder. Then insert the following statements into index.html:

<!DOCTYPE html>
<html lang="en" ng-app="myApp">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Intellipix</title>
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.6/css/bootstrap.min.css" rel="stylesheet">
<style>
    .ng-cloak { display: none !important; }
    a img {
    	cursor: pointer;
    }
    	.image-modal img {
    width: 100%;
    }
</style>
</head>
<body class="ng-cloak">

<div class="container body-content" ng-controller="mainCtrl as ctrl">

    <h1>Intellipix</h1>

    <!-- Panel containing image-upload and search controls -->
    <div class="well">
    <form ng-if="!ctrl.analysis.inProgress">
        <div class="row">
        <div class="col-md-7">
            <div class="form-group">
            <label for="imageFile">Select Image to Analyze:</label>
            <input type="file" id="imageFile" name="imageFile" on-file-selected="ctrl.imageFileSelected(file)">
            </div>
        </div>
        <div class="col-md-5">
            <div class="form-group">
            <label for="searchText">Search:</label>
            <div class="input-group">
                <input type="text" id="searchText" ng-model="ctrl.searchText" class="form-control">
                <span class="input-group-btn">
                <button class="btn btn-default" type="button" ng-click="ctrl.clearSearchText()">
                    <span class="glyphicon glyphicon-remove" aria-hidden="true"></span>
                </button>
                </span>
            </div>
            </div>
        </div>
        </div>
    </form>
    <p ng-if="ctrl.analysis.inProgress">
        <span class="glyphicon glyphicon-time" aria-hidden="true"></span>
        Analyzing Image ...
    </p>
    </div>

    <!-- Panel showing error message if upload or image analysis fails -->
    <div class="alert alert-danger alert-dismissible" role="alert" ng-if="ctrl.analysis.error">
    <button type="button" class="close" data-dismiss="alert" aria-label="Close"><span aria-hidden="true">&times;</span></button>
    <span class="glyphicon glyphicon-exclamation-sign" aria-hidden="true"></span>
    {{ctrl.analysis.error.message}}
    </div>

    <!-- Thumbnail images -->
    <div class="row">
    <div class="col-sm-12">
        <a ng-click="ctrl.showImageDetails(img)" ng-repeat="img in ctrl.images | filter:ctrl.imageFilter">
            <img ng-src="{{img.url}}" width="192" ng-attr-title={{img.metadata.caption}} style="padding-right: 16px; padding-bottom: 16px">
        </a>
    </div>
    </div>

    <!-- Modal window used to show enlarged images -->
    <div class="modal fade image-modal" id="imageModal" tabindex="-1" role="dialog" aria-labelledby="myModalLabel">
    <div class="modal-dialog modal-lg" role="document">
        <div class="modal-content">
        <div class="modal-header">
            <button type="button" class="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
            <h4 class="modal-title" id="myModalLabel">{{ctrl.current.metadata.caption}}</h4>
        </div>
        <div class="modal-body">
            <img ng-src="{{ctrl.current.fullUrl}}">
        </div>
        </div>
    </div>
    </div>
</div>

<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/3.3.6/js/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.6/angular.min.js"></script>
<script src="index.js"></script>

</body>
</html>

This is the HTML file containing the site’s one and only page. It uses Bootstrap’s grid layout system to align elements on the page, and it uses AngularJS to make the page dynamic. Notice the   ng- attributes such as ng-click and ng-src attached to some of the page’s elements, as well as the “mustache” expressions in double curly braces (for example, {{ctrl.current.metadata.caption}}). These attributes and expressions are part of AngularJS and are frequently found in pages that use it.

17. Add a file named index.js to the “src” folder and insert the following statements:

(function() {

    function mainController($http) {
        this.$http = $http;
        this.analysis = {
            inProgress: false
        };
        this.images = [];
        this.current = null;
        this.searchText = '';
        this.imageFilter = imageFilter.bind(this);
        this.loadImageList();

        function imageFilter(img) {
            var search = this.searchText;
            var tags = img && img.metadata && img.metadata.tags;

            if(!search || !tags) {
                return true;
            }

            if(containsText(tags, search)) {
                return true;
            }

            return false;
        }
    }
    mainController.prototype = {

        clearSearchText: function() {
            this.searchText = '';
        },

        imageFileSelected: function(file) {
            var ctrl = this, formData;
            ctrl.analysis = {
                inProgress: true
            };
            formData = new FormData();
            formData.append('imageFile', file);
            ctrl.$http
                .post('/api/image-upload', formData, {
                    transformRequest: angular.identity,
                    headers: {
                        'Content-Type': undefined
                    }
                })
                .then(function(result) {
                    ctrl.analysis = {
                        inProgress: false,
                        result: result.data
                    };
                    ctrl.loadImageList();
                })
                .catch(function(err) {
                    ctrl.analysis = {
                        inProgress: false,
                        error: err.data || { message: err.statusText }
                    };
                });
        },

        loadImageList: function() {
            var ctrl = this;
            ctrl.$http.get('/api/images')
                .then(function(result) {
                    ctrl.images = result.data.entries || [];
                })
                .catch(function(err) {
                    alert((err.data && err.data.message) || err.toString());
                });
        },

        showImageDetails: function(img) {
            this.current = img;
            angular.element("#imageModal").modal();
        }
    };

    function fileContentBinderDirective() {
        return {
            restrict: 'A',
            scope: {
                onFileSelected: '&'
            },
            link: function(scope, element) {
                element.on('change', function() {
                    scope.$apply(function() {
                        scope.onFileSelected({
                            file: element[0].files[0]
                        });
                    });
                });
            }
        };
    }

    function containsText(text, search) {
        return (text && search) ? (text.toLowerCase().indexOf(search.toLowerCase()) > -1) : false;
    }

    angular
        .module('myApp', [])
        .controller('mainCtrl', ['$http', mainController])
        .directive('onFileSelected', [fileContentBinderDirective]);

}());

This file contains JavaScript code that runs on the client. Among other things, it provides support for uploading images from the browser, displaying and enlarged version of an image when the image thumbnail is clicked, and filtering the thumbnails shown on the page when the user types in the search box. Much of this is wrapped in an AngularJS controller which manages the flow of data in an AngularJS application.

18. Use Visual Studio Code’s File -> Save All command to save all of your changes.

19. Click the Source Control button in the activity bar. Type “First commit” (without quotation marks) into the message box, and then click the Commit button (the check mark) to commit all changes to the local Git repository.

With the code that comprises the app in place and key environment variables initialized with “secrets” such as your storage account key, the next task is to run the app locally and test it in your browser.

 

Leave a Reply

Your email address will not be published. Required fields are marked *