skip to Main Content

How do I draw received png data from ajax call and insert into canvas?

Video file (mp4) is uploaded via ajax to flask server. First frame is extracted with Cv2, saved and the file returned to the client.

I can get png data as string, but want to display image in a canvas.

I can’t inject the code by using image.src = “data:image/PNG;base64″+data. Unsure the source of the error that occurs.

EDIT
I am able to pass image data to an img element, it only shows an icon and not the image itself.

flask functions – main

@app.route('/',methods=['GET','POST'])
def request_from_main():
    if request.method == 'POST' and 'file' in request.files:
       file_object = request.files['file']
       secure_file_object = secure_filename(file_object.filename)
       file_object.save(f"uploads/{secure_file_object}")

       video_container = video_processing.process_saved_video_file(f"uploads/{secure_file_object}")

       saved_image_file_path,saved_image_file_name = video_container.save_image()
       return send_file(saved_image_file_path,mimetype='image/png',as_attachment=True)

       #return video_container.base_64_encode()
   elif request.method == 'GET':
       return render_template('index.html')
   else:
       return None

flask function – cv2 save and provide file path

    def save_image(self): # TODO add error checking for image
    cv2.imwrite(f"{self.file_structure}/{self.base_file_name}.png",self.current_image)
    return f"{self.file_structure}/{self.base_file_name}.png", f"{self.base_file_name}.png"

Client – JavaScript

<!-- language: lang-js -->
// Upload class
var Upload = function (file) {
    this.file = file;
};
Upload.prototype.getName = function() {
    return this.file.name;
};
Upload.prototype.doUpload = function () {
    var that = this;
    var formData = new FormData();

    // add assoc key values, this will be posts values
    formData.append("file", this.file, this.getName());
    formData.append("upload_file", true);

    // Change url according to flask design
    $.ajax({
        type: "POST",
        url: "/", 
        xhr: function () {
            var myXhr = $.ajaxSettings.xhr();
            if (myXhr.upload) {
                myXhr.upload.addEventListener('progress', that.progressHandling, false);
            }
            return myXhr;
        },
        success: function (data) {

            console.log("SUCCESS, pushing to canvas");
            console.log(typeof data);
            $("#temp_storage_image").attr("src",data);
        },
        error: function (error) {
            console.log("ERROR")
            console.log(error);
        },
        async: true,
        data: formData, // What to send
        cache: false,
        contentType: 'image/png',
        //processData: false,
        dataType: 'image/png', // What to expect back from the server, must match otherwise error is thrown,
        timeout: 100000
    });
};

//Change id to your id
$("#file_upload_input").on("change", function (e) {
    var file = $(this)[0].files[0];
    console.log("Made it here, to on change file upload");
    var upload = new Upload(file);
    console.log("Made it past new Upload");
    // maby check size or type here with upload.getSize() and upload.getType()

    // execute upload
    upload.doUpload();
});

Client – HTML

<body>
  <div class="container">
    <div id="project_structure_menu">

      <form id="file_upload" idmethod="POST" enctype="multipart/form-data" action="{{url_for('request_from_main')}}"> <!-- request_from_main is flask function -->
        <div class="input-field col-sm">
          <input id="file_upload_input" type="file" name="file">
        </div>
        <div class="input-field col-sm">
          <button id="file_upload_submit_button" type="submit">Submit</button>
      </div>
      </form>

      <div id="progress-wrp" class="col s2">
        <div class="progress-bar"></div>
        <div class="status">0%</div>
      </div>

    </div>


    <div id="labeling_ui" class="row">
     <img id="temp_storage_image"></img>
     <canvas id="main_drawing" width="100" height="100" style="border:1px solid #d3d3d3;"></canvas>
    </div>
  </div>  

  <!-- TODO Make own file: Get form data and send it to server -->
  <script type="text/javascript" src="{{url_for('static',filename='scripts/file_upload.js')}}"></script> <!-- AJAX file upload -->
</body>

2

Answers


  1. Chosen as BEST ANSWER

    Solved it! My lawd.

    How to send a video file through ajax, process video and return first frame in an img tag

    1. GET request to receive index.html, .js, .css etc
    2. AJAX to send video file to Flask server through POST request
    3. Flask to process video file with cv2 and save image. Return image through send_file.
    4. Insert img tag into "#hiddendiv", point src to collect image through GET request.
    5. Use canvas.getContext('2d).drawImage(...)

    This solution sends identifier string to collect file through get request. Unsure of how secure this solution is.

    Flask Code

    @app.route('/',methods=['GET','POST'])
    def request_from_main():
        if request.method == 'POST' and 'file' in request.files:
        file_object = request.files['file']
        secure_file_object = secure_filename(file_object.filename)
        file_object.save(f"uploads/{secure_file_object}")
    
        video_container = video_processing.process_saved_video_file(f"uploads/{secure_file_object}")
    
            saved_image_file_path,saved_image_file_name = video_container.save_image()
            return  send_file(saved_image_file_path)
        elif request.method == 'GET':
            return render_template('index.html')
        else:
            return None
    # What the identifier string is passed to and retrieve the image to send with send_file
    @app.route('/processed_images',methods=['GET'])
    def new_data_request():
        if request.method == 'GET':
            temp = request.args
            file_name = request.args.get('file')
            return send_file(f"{os.getcwd()}/uploads/{file_name}")
    

    Video Processing

    class video_file_container():
        '''
            Stores the video path in memory so that it can be retrieved and images served to client
        '''
        file_structure = f"{os.getcwd()}/uploads"
        def __init__(self,video_file_path:str,video_capture:cv2.VideoCapture,first_image):             # TODO add error checking for  
            if type(video_file_path) != str:
                raise TypeError(f"ERROR[video_file_container.__init__]: video_file_path is not of type str instead it is {type(video_file_path)}")
            if os.path.exists(video_file_path) == False:
                raise ValueError(f"ERROR[video_file_container.__init__]: video_file_path=|{video_file_path}| does not exist")
            if type(video_capture) != cv2.VideoCapture:
                raise TypeError(f"ERROR[video_file_meta_data.__init__]: video_capture is not of type cv2.VideoCapture instead it is {type(video_capture)}")
    
            self.location = video_file_path
            self.video_file_name = os.path.basename(video_file_path)
            self.base_file_name = os.path.splitext(self.video_file_name)[0]
            self.number_of_frames = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
            self.frame_width = int(video_capture.get(cv2.CAP_PROP_FRAME_WIDTH))
            self.frame_height = int(video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
            self.video_fps  = video_capture.get(cv2.CAP_PROP_FPS)
            self.current_frame = 0
            self.current_image = first_image
    
        def save_image(self): # TODO add error checking for image
            successful_write = cv2.imwrite(f"{self.file_structure}/{self.base_file_name}.png",self.current_image)
            if successful_write is True:
                return f"processed_images/{self.base_file_name}.png", f"{self.base_file_name}.png"
            return None
    

    JavaScript - AJAX - jQuery

    // https://stackoverflow.com/questions/2320069/jquery-ajax-file-upload 
    // Upload class
    var Upload = function (file) {
        this.file = file;
    };
    
    Upload.prototype.getSize = function() {
        return this.file.size;
    };
    Upload.prototype.getName = function() {
        return this.file.name;
    };
    
    // https://stackoverflow.com/questions/35848573/ajax-response-image-element-onto-canvas
    
    Upload.prototype.doUpload = function () {
        var that = this;
        var formData = new FormData();
    
        // add associated key values, this will be posts values
        formData.append("file", this.file, this.getName());
        formData.append("upload_file", true);
    
        // Change url according to flask design
        $.ajax({
            type: "POST",
            url: "/", 
            success: function (data) {
                console.log("received successful load");
                $('#hiddendiv').html(data);
                console.log(document.getElementById("hiddendiv"));
                var img=new Image();
    
                // The path to receive the image, might have to do original way
                img.src="processed_images?file=test_video_single_plant.png";
    
                console.log(img);
                $('#hiddendiv').hide();
    
                var canvas=document.getElementById("main_drawing_canvas");
                var canvas_context=canvas.getContext("2d");
                img.onload= function () {
                    console.log("img.onload function successfully read")
                // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage
                    var source_start_x = 600;  
                    var source_start_y = 600; 
                    var source_width_x = 100;
                    var source_width_y = 100;
                    var destination_start_x = 0;
                    var destination_start_y = 0;
                    var destination_width_x = 100;
                    var destination_width_y = 100;
    
                    canvas_context.drawImage(img
                        ,source_start_x
                        ,source_start_y
                        ,source_width_x
                        ,source_width_y
                        ,destination_start_x
                        ,destination_start_y
                        ,destination_width_x
                        ,destination_width_y
                    );
                }
            },
            error: function (error) {
                console.log("ERROR uploading ajax");
            },
            async: true,
            data: formData, // What to send
            cache: false,
            contentType: false,
            processData: false,
            timeout: 100000 // How long to wait until time out
        });
    };
    

    Index - HTML

    <div class="container">
    <div id="project_structure_menu">
    
      <form id="file_upload" idmethod="POST" enctype="multipart/form-data" action="{{url_for('request_from_main')}}"> <!-- request_from_main is flask function -->
        <div class="input-field col-sm">
          <input id="file_upload_input" type="file" name="file">
        </div>
        <div class="input-field col-sm">
          <button id="file_upload_submit_button" type="submit">Submit</button>
      </div>
      </form>
    
      <div id="progress-wrp" class="col s2">
        <div class="progress-bar"></div>
        <div class="status">0%</div>
      </div>
    
    </div>
    
    
    <div id="labeling_ui" class="row">
      <canvas id="main_drawing" width="1920" height="1080" style="border:1px solid #d3d3d3;"></canvas>
    </div>
    <div id="hidendiv"></div>
    


  2. (1) Flask server image recv

    import flask
    import io
    from PIL import Image
    app = flask.Flask(__name__)
    
    @app.route("/upload", methods=["POST"])
    def predict():
        data = {"success": False}
        if flask.request.method == "POST":
            if flask.request.files.get("image"):
                image = flask.request.files["image"].read()
                image = Image.open(io.BytesIO(image))
                print(image.format, image.size, image.mode)
                data = {"success": True}
    
        res = flask.jsonify(data)
        res.headers["Access-Control-Allow-Origin"] = "*"
        return res
    

    (2) jQuery canvas image send

    async function saveImage(canvas) {
        var imgDataUrl = canvas.toDataURL('image/jpeg');
    
        var blobBin = atob(imgDataUrl.split(',')[1]);    // base64 데이터 디코딩
        var array = [];
        for (var i = 0; i < blobBin.length; i++) {
            array.push(blobBin.charCodeAt(i));
        }
    
        var file = new Blob([new Uint8Array(array)], {type: 'image/jpeg'});    // Blob 생성
    
        var formdata = new FormData();    // formData 생성
        formdata.append("image", file);    // file data 추가
        return formdata
    }
    
    async function request_predict(canvas){
        var data = await saveImage(canvas)
        $.ajax({
            type: "POST",
            enctype: 'multipart/form-data',
            url: "http://localhost:port/upload",
            data: data,
            processData: false,
            contentType: false,
            cache: false,
            timeout: 600000,
            success: function (data) {
                console.log("data : ", data);
            },
            error: function (e) {
                console.log("ERROR : ", e);
            }
        });
    }
    
    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search