Nginx - Extract the data from the body of a POST request from the browser to a Bash server with Ncat

karim_mo
January 3, 2025
73 views
1 vote
2 Answers

I want to extract the data of the body of a POST request from the browser to a Bash server with Ncat using only Bash Scripting, I have tried AI tools and searched and tried for days but I can’t access the data, I am new to Bash Scripting.

Most of the time the browser and the server freeze probably due to an infinite loop or problem in the htttp response despite the presence of both in the code.

I would really appreciate if you can demonstrate with a code block on how to extract all the data needed from a POST request coming from the browser to a Bash Server that uses Ncat such as the http method, route, data of the body request…etc, thanks in advance.

Here is one of my many trials:

#!/bin/bash

PORT=8080
DIR="/var/www/html"

setup_nginx(){
    sudo apt-get update
    sudo apt-get install -y nginx
}

configure_nginx(){

    cat > /etc/nginx/sites-enabled/default << EOF
        server {
            listen 80;
            server_name localhost;
    
            root /var/www/html;
            index index.html index.htm index.nginx-debian.html;

            # Timeouts
            proxy_connect_timeout 60;
            proxy_read_timeout 60;
            proxy_send_timeout 60;
            
            # Buffer settings
            proxy_buffering off;
            proxy_request_buffering off;

            location / {
                proxy_pass http://localhost:$PORT;
            }
        }
EOF

    sudo nginx -t && systemctl reload nginx
}

setup_ncat(){
    sudo apt-get update
    sudo apt-get install -y ncat
}

create_HTML_page(){

    cat > $DIR/index.html << EOF
    <!DOCTYPE html>
    <html>
    <head>
        <title>Bash Web Application</title>
    </head>
    <body>
        <h1>Save Messages</h1>
        <form action="/messages" method="POST">
            <input type="text" name="message" placeholder="Enter your message">
            <input type="submit" value="Submit">
        </form>
        <br/>
        <a href="http://localhost:$PORT/messages">Messages Page</a>
        <br/>
        <a href="http://localhost:$PORT/deleteAll">Delete All Messages</a>
    </body>
    </html>
EOF

    cat > $DIR/success.html << EOF
    <!DOCTYPE html>
    <html>
    <head>
        <title>Success</title>
    </head>
    <body>
        <h1>Your operation has been done successfully</h1>
        <a href="http://localhost:$PORT">Home Page</a>
        <br/>
        <a href="http://localhost:$PORT/messages">Messages Page</a>
        <br/>
        <a href="http://localhost:$PORT/deleteAll">Delete All Messages</a>
    </body>
    </html>
EOF
}


# File to store messages
DATA_FILE="$DIR/messages.txt"

# Ensure the data file exists
if [ ! -f "$DATA_FILE" ]; then
    cd "$DIR"
    touch messages.txt
fi


create_server(){
    echo "Starting web server on port $PORT..."
    echo "Visit http://localhost:$PORT to view the website."

    while true; do

        # Listen for connections and process requests
        ncat -l -p $PORT -c '
            
            DIR=/var/www/html
            set -x

            # Read the first line of the HTTP request (e.g., POST /endpoint HTTP/1.1)
            read -r REQUEST_FIRST_LINE
            METHOD=$(echo "$REQUEST_FIRST_LINE" | cut -d" " -f1)
            ROUTE=$(echo "$REQUEST_FIRST_LINE" | cut -d" " -f2)
            
            REQUEST=""           
            

            if [ "$METHOD" = "POST" ]; then
                while IFS= read -r line; do
                    # Extract Content-Length if present
                    if echo "$line" | grep -q "^Content-Length:"; then
                        CONTENT_LENGTH=$(echo "$line" | cut -d" " -f2)
                    fi

                    line=${line%%$'r'}
                    REQUEST+="$line"$'n'

                    if [ "$line" = " " ]; then
                        # If we have Content-Length, read the body
                        if [ ! -z "$CONTENT_LENGTH" ]; then
                            # Read the POST body
                            body=$(dd bs=1 count=$CONTENT_LENGTH 2>/dev/null)
                            
                            # Add body to complete request
                            REUQEST+="$body"
                        fi
                    break
                    fi

                done
                
            fi


            # Determine which file to serve based on the requested route
            case "$ROUTE" in
                "/")
                    if [ "$METHOD" = "GET" ]; then    
                        FILE="$DIR/index.html"
                        STATUS="200 OK"
                    fi
                    ;;
                "/messages")
                    if [ "$METHOD" = "GET" ]; then    
                        FILE="$DIR/messages.txt"
                        STATUS="200 OK"
                        today=$(date +"%Y-%m-%d")
                    
                    elif [ "$METHOD" = "POST" ]; then
                    
                        read -r request
                        # data=$(echo "$request" | sed -e "s/+/ /g" -e "s/.*message=([^&]*).*/1/")
                        data= $(echo "$request")
                        echo "$data" >> $DIR/messages.txt
                        FILE="$DIR/success.html"
                        STATUS="200 OK"
                        today=$(date +"%Y-%m-%d")
                    fi
                    ;;
                "/deleteAll")
                    if [ "$METHOD" = "GET" ]; then
                        > $DIR/messages.txt
                        FILE="$DIR/success.html"
                        STATUS="200 OK"
                        today=$(date +"%Y-%m-%d")
                    fi
                    ;;
                *)
                    FILE="templates/404.html"
                    STATUS="404 Not Found"
                    ;;
            esac
            
            # Check if the requested file exists
            
            if [ -f "$FILE" ]; then
                RESPONSE_BODY=$(cat "$FILE")
                RESPONSE_BODY=$(eval "echo "$RESPONSE_BODY"")
            else
                RESPONSE_BODY="<html><body><h1>404 Not Found</h1></body></html>"
            fi

            # Send the HTTP response headers and body
            echo "HTTP/1.1 $STATUSr"
            echo "Content-Type: text/htmlr"
            echo "Connection: closer"
            echo "r"  # This is the required blank line separating headers from the body
            echo "$RESPONSE_BODY"
        '

        if [[ "$?" -ne 0 ]]; then
            echo "Error happened during connection"; 
            exit 1;
        fi
    done
}


# handle_http_request(){
    
#     while true; do
#     echo "Listening on port $PORT..."
    
#     # Use netcat to listen for incoming connections
#     {
#         # Read the request line
#         read request
#         log_request "$request"

#         # Extract HTTP method and resource path
#         method=$(echo "$request" | awk '{print $1}')
#         resource=$(echo "$request" | awk '{print $2}')

#         # Process GET request
#         if [[ $method == "GET" ]]; then
#             echo -e "HTTP/1.1 200 OKrnContent-Type: text/plainrnrnGET request received: $resource"
        
#         # Process POST request
#         elif [[ $method == "POST" ]]; then
#             # Read Content-Length header
#             while read header; do
#                 [[ $header == $'r' ]] && break
#                 if [[ $header == Content-Length:* ]]; then
#                     length=${header#Content-Length: }
#                 fi
#             done

#             # Read the payload
#             read -n $length payload
#             echo "$payload" >> "$DATAFILE"

#             echo -e "HTTP/1.1 200 OKrnContent-Type: text/plainrnrnPOST data saved: $payload"

#         # Handle unknown methods
#         else
#             echo -e "HTTP/1.1 405 Method Not AllowedrnContent-Type: text/plainrnrnUnsupported HTTP method: $method"
#         fi
#     } | nc -l -p "$PORT" -q 1
# done
# }


setup(){

    command -v nginx
    if [[ "$?" -ne 0 ]]; then
        echo "nginx is not installed, it will be installed now"
        setup_nginx
    fi

    command -v ncat
    if [[ "$?" -ne 0 ]]; then
        echo "ncat is not installed, it will be installed now"
        setup_ncat
    fi

    configure_nginx
    create_HTML_page
    create_server
}

setup

Answers

Preamble

Ok, for fun and to show how this could work:

Don’t ever do that!

This is only for educational purpose! Without any consideration of security issues!!

Some remarks:

This is a bash script, but inner script have to be checked as POSIX Shell!
This use a loop, you could exit by reaching: http://localhost:8040/quit
The 1st line of inner script is set -x for debugging. you could drop this.
My .html files don’t refer to http://localhost:$PORT/, just /.

My sample

#!/bin/bash

PORT=8040
DIR="/tmp/www"

mkdir -p "$DIR"
cat >"$DIR/success.html" <<-EOSuccess
        <!DOCTYPE html>
        <html>
        <head>
            <title>Success</title>
        </head>
        <body>
            <h1>Your operation has been done successfully</h1>
            <a href="/">Home Page</a>
            <br/>
            <a href="/messages.txt">Messages Page</a>
            <br/>
            <a href="/deleteAll">Delete All Messages</a>
        </body>
        </html>
        EOSuccess

cat >"$DIR/index.html" <<-EOIndex
        <!DOCTYPE html>
        <html>
        <head>
            <title>Bash Web Application</title>
        </head>
        <body>
            <h1>Save Messages</h1>
            <form action="/messages.txt" method="POST">
                <input type="text" name="message" placeholder="Enter your message">
                <input type="submit" value="Submit">
            </form>
            <br/>
            <a href="/messages.txt">Messages Page</a>
            <br/>
            <a href="/deleteAll">Delete All Messages</a>
        </body>
        </html>
        EOIndex

touch "$DIR/messages.txt"
[[ -f $DIR/quit ]] && rm "$DIR/quit"

while [[ ! -f $DIR/quit ]]; do
  DIR=$DIR nc.traditional -lp $PORT -c  $'

    set -x;
    sendMsg() {
        if [ -n "$1" ]; then
            ftype="$(echo "$1" | file --mime-type -b -)"
        else
            ftype=text/plain
        fi
        printf "HTTP/1.0 %s\r\n" "${2:-200 OK}";
        printf "%s: %s\r\n" Date: "$(LANG=C date +"%a, %d %b %Y %T %Z")" 
          Server: shell-script Accept-Ranges bytes Content-Length ${#1} 
          Connection close Content-Type "$ftype";
        printf "\r\n%s" "$1";
    }
    sendErr() {
        sendMsg "<html><body><h1>$1</h1></body></html>" "$1" 
    }
    IFS="r " read -r method path htver _;
    while IFS="r:" read -r fld val _ && [ -n "$fld" ]; do
        case $fld in Content-Length )
            clen=$val ;;
        esac;
    done;

    case $path in
        /|/index.* ) sendMsg "$(cat $DIR/index.html)" ;;
        /stat ) sendMsg "$(cat /proc/loadavg)" ;;
        /messages.txt )
            case $method in
                GET ) sendMsg "$(cat $DIR/messages.txt)" ;;
                POST )
                    if [ -n "$clen" ]; then
                        message="$(head -c "$clen")";
                        message="${message#message=}";
                        echo "$message" | tr + " " >>"$DIR/messages.txt";
                    fi
                    sendMsg "$(cat $DIR/success.html)" ;;
                * ) sendErr "400 Bad Request";;
            esac ;;
        /deleteAll )
                sendMsg "$(cat $DIR/success.html)";
                > $DIR/messages.txt ;;
        /quit )
                sendErr "410 Gone" ;
                echo >"$DIR/quit" ;;
        * ) sendErr "404 Not Found" ;;
    esac
  '

done

Care!

bash use t for inline text not spaces!

Once copied this on a "nc-Server.sh" file, to avoid message like:

 ./nc-Server.sh: line 97: warning: here-document at line 7 delimited by end-of-file

you have to

sed -e 's/        /o11/g' -i nc-Server.sh
chmod +x nc-Server.sh

before executing them!!

I tested with 2 different clients:

A small script using ncat (but failing to add r after each line)

#!/bin/bash
  ncat --no-shutdown localhost 8080 <<@
POST /messages HTTP/1.1
Host: example.com
Content-Type: application/json
Content-Length: 50

{
  "username": "johndoe",
  "password": "1234"
}
@

curl command from prompt

curl http://localhost:8080/messages -H "Content-Type: application/json" --data '{"username": "johndoe", "password": "1234"}'

When removing the r fails, the first program might work while the second fails.

I tried to make minimal changes to your original script:

I changed the indent to 2 spaces
I marked code changes with # Changed
I did not change all variables to lowercase
I did not use printf as a replacement for the echo commands.
I did not use the variable DATA_FILE that you set somewhere.
I did not try to delete unused code

The script in the ncat part is not bash, some commands are not available.
When you write line=${line%%$'r'}, the first single quote matches the opening quote in ncat -l -p $PORT -c ', so removing the r fails
An assignment like data= $(echo "$request") won’t work because of the space after the =.

See the changed code beneath, and look at the lines marked with Changed.

#!/bin/bash

PORT=8080
DIR="/var/www/html"

setup_nginx(){
  sudo apt-get update
  sudo apt-get install -y nginx
}

configure_nginx(){

  # Changed: Added sudo creating and chmodding file
  sudo touch /etc/nginx/sites-enabled/default
  sudo chmod 666 /etc/nginx/sites-enabled/default
  cat > /etc/nginx/sites-enabled/default << EOF
    server {
      listen 80;
      server_name localhost;

      root /var/www/html;
      index index.html index.htm index.nginx-debian.html;

      # Timeouts
      proxy_connect_timeout 60;
      proxy_read_timeout 60;
      proxy_send_timeout 60;

      # Buffer settings
      proxy_buffering off;
      proxy_request_buffering off;

      location / {
        proxy_pass http://localhost:$PORT;
      }
    }
EOF
  sudo nginx -t && systemctl reload nginx
}

setup_ncat(){
  sudo apt-get update
  sudo apt-get install -y ncat
}

create_HTML_page(){

  # Changed: Added sudo
  sudo touch $DIR/index.html
  sudo chmod 666 $DIR/index.html
  cat > $DIR/index.html << EOF
  <!DOCTYPE html>
  <html>
  <head>
    <title>Bash Web Application</title>
  </head>
  <body>
    <h1>Save Messages</h1>
    <form action="/messages" method="POST">
      <input type="text" name="message" placeholder="Enter your message">
      <input type="submit" value="Submit">
    </form>
    <br/>
    <a href="http://localhost:$PORT/messages">Messages Page</a>
    <br/>
    <a href="http://localhost:$PORT/deleteAll">Delete All Messages</a>
  </body>
  </html>
EOF

  # Changed: Added sudo
  sudo touch $DIR/success.html
  sudo chmod 666 $DIR/success.html
  cat > $DIR/success.html << EOF
  <!DOCTYPE html>
  <html>
  <head>
    <title>Success</title>
  </head>
  <body>
    <h1>Your operation has been done successfully</h1>
    <a href="http://localhost:$PORT">Home Page</a>
    <br/>
    <a href="http://localhost:$PORT/messages">Messages Page</a>
    <br/>
    <a href="http://localhost:$PORT/deleteAll">Delete All Messages</a>
  </body>
  </html>
EOF
}


# File to store messages
DATA_FILE="$DIR/messages.txt"

# Ensure the data file exists
if [ ! -f "$DATA_FILE" ]; then
  cd "$DIR"
  # Changed: Added sudo
  sudo touch messages.txt
fi


create_server(){
  echo "Starting web server on port $PORT..."
  echo "Visit http://localhost:$PORT to view the website."

  while true; do

    # Listen for connections and process requests
    ncat -l -p $PORT -c '

      DIR=/var/www/html
      # Changed: removed the `set -x`, which was a good approach for debugging

      # Read the first line of the HTTP request (e.g., POST /endpoint HTTP/1.1)
      read -r REQUEST_FIRST_LINE
      METHOD=$(echo "$REQUEST_FIRST_LINE" | cut -d" " -f1)
      # Changed: Remove r from ROUTE
      ROUTE=$(echo "$REQUEST_FIRST_LINE" | cut -d" " -f2 | tr -d "r")

      REQUEST=""

      if [ "$METHOD" = "POST" ]; then
        while read -r line; do
          # Extract Content-Length if present
          if echo "$line" | grep -q "^Content-Length:"; then
            # Changed: Remove r from ROUTE
            CONTENT_LENGTH=$(echo "$line" | cut -d" " -f2 | tr -d "r")
          fi
          # Changed: delete r without use of single quotes
          line=$(echo "${line}" | tr -d "r")
          # Changed: Added debug line and commented that line.
          # echo >&2 "Line: [${line}], ${#line} chars long"

          # Changed: The '+=' does not work fine here
          REQUEST="${REQUEST}${line}n"

          # Changed: Do not compare $line with a space, it should be empty
          if [ -z "$line" ]; then
            # If we have Content-Length, read the body
            if [ ! -z "$CONTENT_LENGTH" ]; then
              # Read the POST body
              body=$(dd bs=1 count=$CONTENT_LENGTH 2>/dev/null)
              # Changed: Debug show body
              echo >&2 "Body=[$body]"

              # Add body to complete request
              # Changed: Typo REUQEST changed in REQUEST and replaced += operator
              REQUEST="${REQUEST}${body}n"
            fi
            # Changed: Added indent
            break
          fi

        done

      fi

      # Changed: write $REQUEST (never used) to console
      echo >&2 "Request:n${REQUEST}"

      # Determine which file to serve based on the requested route
      case "$ROUTE" in
        "/")
          if [ "$METHOD" = "GET" ]; then
            FILE="$DIR/index.html"
            STATUS="200 OK"
          fi
          ;;
        "/messages")
          if [ "$METHOD" = "GET" ]; then
            FILE="$DIR/messages.txt"
            STATUS="200 OK"
            today=$(date +"%Y-%m-%d")

          elif [ "$METHOD" = "POST" ]; then

            # Changed: Replaced "read -r request" and assigning $request to $data with data=${body}
            data="${body}"
            # data=$(echo "$request" | sed -e "s/+/ /g" -e "s/.*message=([^&]*).*/1/")
            echo "$data" >> $DIR/messages.txt
            FILE="$DIR/success.html"
            STATUS="200 OK"
            today=$(date +"%Y-%m-%d")
          fi
          ;;
        "/deleteAll")
          if [ "$METHOD" = "GET" ]; then
            > $DIR/messages.txt
            FILE="$DIR/success.html"
            STATUS="200 OK"
            today=$(date +"%Y-%m-%d")
          fi
          ;;
        *)
          FILE="templates/404.html"
          STATUS="404 Not Found"
          ;;
      esac

      # Check if the requested file exists

      if [ -f "$FILE" ]; then
        RESPONSE_BODY=$(cat "$FILE")
        RESPONSE_BODY=$(eval "echo "$RESPONSE_BODY"")
      else
        RESPONSE_BODY="<html><body><h1>404 Not Found</h1></body></html>"
      fi

      # Send the HTTP response headers and body
      # Changed: added next comment
      # When you dont want n in the output, use printf
      echo "HTTP/1.1 $STATUSr"
      echo "Content-Type: text/htmlr"
      echo "Connection: closer"
      # Changed: added content-length
      echo "Content-Length: ${#RESPONSE_BODY}r"
      echo "r"  # This is the required blank line separating headers from the body
      echo "$RESPONSE_BODY"
    '

    if [[ "$?" -ne 0 ]]; then
      echo "Error happened during connection";
      exit 1;
    fi
  done
}


# handle_http_request(){

#   while true; do
#   echo "Listening on port $PORT..."

#   # Use netcat to listen for incoming connections
#   {
#     # Read the request line
#     read request
#     log_request "$request"

#     # Extract HTTP method and resource path
#     method=$(echo "$request" | awk '{print $1}')
#     resource=$(echo "$request" | awk '{print $2}')

#     # Process GET request
#     if [[ $method == "GET" ]]; then
#       echo -e "HTTP/1.1 200 OKrnContent-Type: text/plainrnrnGET request received: $resource"

#     # Process POST request
#     elif [[ $method == "POST" ]]; then
#       # Read Content-Length header
#       while read header; do
#         [[ $header == $'r' ]] && break
#         if [[ $header == Content-Length:* ]]; then
#           length=${header#Content-Length: }
#         fi
#       done

#       # Read the payload
#       read -n $length payload
#       echo "$payload" >> "$DATAFILE"

#       echo -e "HTTP/1.1 200 OKrnContent-Type: text/plainrnrnPOST data saved: $payload"

#     # Handle unknown methods
#     else
#       echo -e "HTTP/1.1 405 Method Not AllowedrnContent-Type: text/plainrnrnUnsupported HTTP method: $method"
#     fi
#   } | nc -l -p "$PORT" -q 1
# done
# }


setup(){

  command -v nginx
  if [[ "$?" -ne 0 ]]; then
    echo "nginx is not installed, it will be installed now"
    setup_nginx
  fi

  command -v ncat
  if [[ "$?" -ne 0 ]]; then
    echo "ncat is not installed, it will be installed now"
    setup_ncat
  fi

  configure_nginx
  create_HTML_page
  create_server
}

setup

Result:

curl http://localhost:8080/messages -H "Content-Type: application/json" --data '{"username": "johndoe", "password": "1234"}'
  <!DOCTYPE html>
  <html>
  <head>
    <title>Success</title>
  </head>
  <body>
    <h1>Your operation has been done successfully</h1>
    <a href=http://localhost:8080>Home Page</a>
    <br/>
    <a href=http://localhost:8080/messages>Messages Page</a>
    <br/>
    <a href=http://localhost:8080/deleteAll>Delete All Messages</a>
  </body>
  </html>

Please signup or login to give your own answer.

Click here to cancel reply.

Nginx – Extract the data from the body of a POST request from the browser to a Bash server with Ncat

Answers

Preamble

Some remarks:

My sample

Care!