How to automate isolating line art from the background with code. Is there a way? – Photoshop

What OP described, I know from GIMP where it is called Color to Alpha.

While I used that command from time to time I tried to imagine how this could be implemented.

Multiple approaches come into my mind:

• a very simple: compare every pixel with a pivot color and set alpha to 0 in case of match
• a threshold-based: determine the Euclidean distance of pixel to pivot color in RGB space (as 3D space) and set alpha according to distance when under a given threshold
• threshold-based in HSV space: the similar approach like above but applied to HSV space (for better color matching).

Out of curiosity, I wrote a sample application to try this out.

First a C++ code for color to alpha transformation:

imageColorToAlpha.h:

#ifndef IMAGE_COLOR_TO_ALPHA_H
#define IMAGE_COLOR_TO_ALPHA_H

// standard C++ header:
#include <cstdint>
#include <functional>

// convenience types
typedef std::uint32_t Pixel;
typedef std::uint8_t Comp;

// convenience constants
const int ShiftR = 16;
const int ShiftG = 8;
const int ShiftB = 0;
const int ShiftA = 24;

const Pixel MaskR = 0xff << ShiftR;
const Pixel MaskG = 0xff << ShiftG;
const Pixel MaskB = 0xff << ShiftB;
const Pixel MaskA = 0xff << ShiftA;

const Pixel MaskRGB = MaskR | MaskG | MaskB;

// convenience functions
inline Comp getR(Pixel pixel) { return Comp(pixel >> ShiftR); }
inline Comp getG(Pixel pixel) { return Comp(pixel >> ShiftG); }
inline Comp getB(Pixel pixel) { return Comp(pixel >> ShiftB); }
inline Comp getA(Pixel pixel) { return Comp(pixel >> ShiftA); }

inline void setR(Pixel &pixel, Comp r)
{
  pixel &= ~MaskR;
  pixel |= r << ShiftR;
}
inline void setG(Pixel &pixel, Comp g)
{
  pixel &= ~MaskG;
  pixel |= g << ShiftG;
}
inline void setB(Pixel &pixel, Comp b)
{
  pixel &= ~MaskB;
  pixel |= b << ShiftB;
}
inline void setA(Pixel &pixel, Comp r)
{
  pixel &= ~MaskA;
  pixel |= r << ShiftA;
}
inline void set(Pixel &pixel, Comp r, Comp g, Comp b)
{
  pixel &= ~MaskRGB;
  pixel |= r << ShiftR | g << ShiftG | b << ShiftB;
}
inline void set(Pixel &pixel, Comp r, Comp g, Comp b, Comp a)
{
  pixel = r << ShiftR | g << ShiftG | b << ShiftB | a << ShiftA;
}

extern void transformImage(
  size_t w, size_t h, // width and height
  size_t bytesPerRow, // bytes per row (to handle row alignment)
  const Pixel *imgSrc, // source image
  Pixel *imgDst, // destination image
  std::function<Pixel(Pixel)> transform);

// color to alpha (very simple)
extern Pixel colorToAlpha(Pixel pixel, Pixel color);

// color to alpha (with threshold)
extern Pixel colorToAlpha(
  Pixel pixel, Pixel color, unsigned threshold);

// convenience functions for image

inline void colorToAlphaSimple(
  size_t w, size_t h, // width and height
  size_t bytesPerRow, // bytes per row (to handle row alignment)
  const Pixel *imgSrc, // source image
  Pixel *imgDst, // destination image
  Pixel color) // pivot color
{
  transformImage(w, h, bytesPerRow, imgSrc, imgDst,
    [color](Pixel pixel) { return colorToAlpha(pixel, color); });
}

inline void colorToAlphaThreshold(
  size_t w, size_t h, // width and height
  size_t bytesPerRow, // bytes per row (to handle row alignment)
  const Pixel *imgSrc, // source image
  Pixel *imgDst, // destination image
  Pixel color, // pivot color
  unsigned threshold) // threshold
{
  transformImage(w, h, bytesPerRow, imgSrc, imgDst,
    [color, threshold](Pixel pixel) {
      return colorToAlpha(pixel, color, threshold);
    });
}

inline void fillRGB(
  size_t w, size_t h, // width and height
  size_t bytesPerRow, // bytes per row (to handle row alignment)
  Pixel *img, // image to modify
  Pixel color) // fill color (alpha ignored)
{
  color &= MaskRGB;
  transformImage(w, h, bytesPerRow, img, img,
    [color](Pixel pixel) {
      pixel &= ~MaskRGB; pixel |= color; return pixel;
    });
}

#endif // IMAGE_COLOR_TO_ALPHA_H

and the corresponding imageColorToAlpha.cc:

// standard C++ header:
#include <cmath>

// header of this module:
#include "imageColorToAlpha.h"

void transformImage(
  size_t w, size_t h, // width and height
  size_t bytesPerRow, // bytes per row (to handle row alignment)
  const Pixel *imgSrc, // source image
  Pixel *imgDst, // destination image
  std::function<Pixel(Pixel)> transform)
{
  for (size_t y = 0; y < h; ++y) {
    const Pixel *pixelSrc = (const Pixel*)((const Comp*)imgSrc + y * bytesPerRow);
    Pixel *pixelDst = (Pixel*)((Comp*)imgDst + y * bytesPerRow);
    for (size_t x = 0; x < w; ++x) pixelDst[x] = transform(pixelSrc[x]);
  }
}

Pixel colorToAlpha(Pixel pixel, Pixel color)
{
  // eliminate current alpha values from pixel and color
  pixel &= MaskRGB; color &= MaskRGB;
  // compare pixel with color
  const int match = pixel == color;
  // set alpha according to match of pixel and color
  setA(pixel, ~(match * 0xff));
  // done
  return pixel;
}

Pixel colorToAlpha(Pixel pixel, Pixel color, unsigned threshold)
{
  // delta values per component
  const int dR = (int)getR(pixel) - (int)getR(color);
  const int dG = (int)getG(pixel) - (int)getG(color);
  const int dB = (int)getB(pixel) - (int)getB(color);
  // square Euclidean distance
  const unsigned dSqr = dR * dR + dG * dG + dB * dB;
  // compute alpha
  Comp a = 0xff;
  if (dSqr < threshold * threshold) { // distance below threshold?
    // compute alpha weighted by distance
    const double d = sqrt((double)dSqr);
    const double f = d / threshold;
    a = (Comp)(f * 0xff);
  }
  // done
  setA(pixel, a);
  return pixel;
}

This image manipulation code is based on the C++ std library only.
This is intended to make the code as exemplary and re-usable as possible.

However, the code for decoding image file formats is often neither short nor simple.
Hence, I wrote a wrapper application in Qt to show this in action.
Qt provides image support as well as the frame work for a desktop application and seemed to me as most appropriate for this task (beside of the fact that I’ve some experience with it).

The Qt wrapper application testQImageColorToAlpha.cc:

// Qt header:
#include <QtWidgets>

// own header:
#include "imageColorToAlpha.h"
#include "qColorButton.h"

// convenience functions
QPixmap fromImage(const QImage &qImg)
{
  QPixmap qPixmap;
  qPixmap.convertFromImage(qImg);
  return qPixmap;
}

QPixmap fromAlphaImage(
  const QImage &qImg,
  QColor qColor1 = Qt::darkGray,
  QColor qColor2 = Qt::gray,
  int whCell = 32)
{
  QPixmap qPixmap(qImg.width(), qImg.height());
  { QPainter qPainter(&qPixmap);
    // draw chessboard
    qPixmap.fill(qColor1);
    for (int y = 0; y < qImg.height(); y += 2 * whCell) {
      for (int x = 0; x < qImg.width(); x += 2 * whCell) {
        qPainter.fillRect(x, y, whCell, whCell, qColor2);
        qPainter.fillRect(x + whCell, y + whCell, whCell, whCell, qColor2);
      }
    }
    // overlay with image
    qPainter.drawImage(0, 0, qImg);
  } // close Qt painter
  // done
  return qPixmap;
}

enum {
  SingleValue,
  RGBRange
};

QImage colorToAlphaSimple(
  const QImage &qImgSrc, QColor qColor,
  bool fill, QColor qColorFill)
{
  // ensure expected format for input image
  QImage qImg = qImgSrc.convertToFormat(QImage::Format_ARGB32);
  const int w = qImg.width(), h = qImg.height(), bpr = qImg.bytesPerLine();
  // allocate storage for output image
  QImage qImgDst(w, h, QImage::Format_ARGB32);
  colorToAlphaSimple(w, h, bpr,
    (const Pixel*)qImg.constBits(), (Pixel*)qImgDst.bits(), qColor.rgba());
  // override RGB if required
  if (fill) fillRGB(w, h, bpr, (Pixel*)qImgDst.bits(), qColorFill.rgba());
  // done
  return qImgDst;
}

QImage colorToAlphaThreshold(
  const QImage &qImgSrc, QColor qColor, unsigned threshold,
  bool fill, QColor qColorFill)
{
  // ensure expected format for input image
  QImage qImg = qImgSrc.convertToFormat(QImage::Format_ARGB32);
  const int w = qImg.width(), h = qImg.height(), bpr = qImg.bytesPerLine();
  // allocate storage for output image
  QImage qImgDst(w, h, QImage::Format_ARGB32);
  colorToAlphaThreshold(w, h, bpr,
    (const Pixel*)qImg.constBits(), (Pixel*)qImgDst.bits(), qColor.rgba(), threshold);
  // override RGB if required
  if (fill) fillRGB(w, h, bpr, (Pixel*)qImgDst.bits(), qColorFill.rgba());
  // done
  return qImgDst;
}

// main application
int main(int argc, char **argv)
{
  qDebug() << "Qt Version:" << QT_VERSION_STR;
  QApplication app(argc, argv);
  // setup data
  QImage qImgIn("cat.drawn.png");
  QImage qImgOut(qImgIn);
  // setup GUI
  // main window
  QWidget qWin;
  qWin.setWindowTitle(QString::fromUtf8("Color to Alpha"));
  QGridLayout qGrid;
  // input image
  QHBoxLayout qHBoxLblIn;
  QLabel qLblIn(QString::fromUtf8("Input Image:"));
  qHBoxLblIn.addWidget(&qLblIn);
  qHBoxLblIn.addStretch(1);
  QPushButton qBtnLoad(QString::fromUtf8("Open..."));
  qHBoxLblIn.addWidget(&qBtnLoad);
  qGrid.addLayout(&qHBoxLblIn, 0, 0);
  QLabel qLblImgIn;
  qLblImgIn.setPixmap(fromImage(qImgIn));
  qGrid.addWidget(&qLblImgIn, 1, 0);
  // config. color to alpha
  QGroupBox qBoxCfg(QString::fromUtf8("Configuration:"));
  QFormLayout qFormCfg;
  QComboBox qMenuColorToAlpha;
  qMenuColorToAlpha.addItem(QString::fromUtf8("Single Value"));
  qMenuColorToAlpha.addItem(QString::fromUtf8("With Threshold"));
  qFormCfg.addRow(QString::fromUtf8("Color to Transparency:"), &qMenuColorToAlpha);
  QColorButton qBtnColor(Qt::white);
  qFormCfg.addRow(QString::fromUtf8("Pivot Color:"), &qBtnColor);
  qBoxCfg.setLayout(&qFormCfg);
  QSpinBox qEditRange;
  qEditRange.setRange(1, 255);
  qFormCfg.addRow(QString::fromUtf8("Range:"), &qEditRange);
  QFrame qHSepCfg;
  qHSepCfg.setFrameStyle(QFrame::HLine | QFrame::Plain);
  qFormCfg.addRow(&qHSepCfg);
  QHBoxLayout qHBoxFill;
  QCheckBox qTglFill;
  qTglFill.setChecked(false);
  qHBoxFill.addWidget(&qTglFill);
  QColorButton qBtnColorFill(Qt::black);
  qHBoxFill.addWidget(&qBtnColorFill, 1);
  qFormCfg.addRow(QString::fromUtf8("Fill Color:"), &qHBoxFill);
  qGrid.addWidget(&qBoxCfg, 1, 1);
  // output image
  QHBoxLayout qHBoxLblOut;
  QLabel qLblOut(QString::fromUtf8("Output Image:"));
  qHBoxLblOut.addWidget(&qLblOut);
  qHBoxLblOut.addStretch(1);
  QColorButton qBtnBgColor1(QString::fromUtf8("Color 1"), Qt::darkGray);
  qHBoxLblOut.addWidget(&qBtnBgColor1);
  QColorButton qBtnBgColor2(QString::fromUtf8("Color 2"), Qt::gray);
  qHBoxLblOut.addWidget(&qBtnBgColor2);
  qGrid.addLayout(&qHBoxLblOut, 0, 2);
  QLabel qLblImgOut;
  qLblImgOut.setPixmap(fromAlphaImage(qImgOut));
  qGrid.addWidget(&qLblImgOut, 1, 2);
  // main window
  qWin.setLayout(&qGrid);
  qWin.show();
  // helper
  auto update = [&]() {
    const int algo = qMenuColorToAlpha.currentIndex();
    switch (algo) {
      case SingleValue:
        qImgOut
          = colorToAlphaSimple(qImgIn, qBtnColor.color(),
            qTglFill.isChecked(), qBtnColorFill.color());
        break;
      case RGBRange:
        qImgOut
          = colorToAlphaThreshold(qImgIn, qBtnColor.color(), qEditRange.value(),
            qTglFill.isChecked(), qBtnColorFill.color());
        break;
    }
    qEditRange.setEnabled(algo >= RGBRange);
    qBtnColorFill.setEnabled(qTglFill.isChecked());
    qLblImgOut.setPixmap(
      fromAlphaImage(qImgOut, qBtnBgColor1.color(), qBtnBgColor2.color()));
  };
  // install signal handlers
  QObject::connect(
    &qBtnLoad, &QPushButton::clicked,
    [&]() {
      QString filePath
        = QFileDialog::getOpenFileName(
          &qWin, QString::fromUtf8("Open Image File"),
          QString(),
          QString::fromUtf8(
            "Image Files (*.png *.jpg *.jpeg);;"
            "PNG Files (*.png);;"
            "JPEG Files (*.jpg *.jpeg);;"
            "All Files (*)"));
      if (filePath.isEmpty()) return; // choice aborted
      QImage qImg;
      qImg.load(filePath);
      if (qImg.isNull()) return; // file loading failed
      qImgIn = qImg;
      qLblImgIn.setPixmap(fromImage(qImgIn));
      update();
    });
  QObject::connect(
    &qMenuColorToAlpha,
    QOverload<int>::of(&QComboBox::currentIndexChanged),
    [&](int) { update(); });
  QObject::connect(&qBtnColor, &QPushButton::clicked,
    [&]() { qBtnColor.chooseColor(); update(); });
  QObject::connect(
    &qEditRange, QOverload<int>::of(&QSpinBox::valueChanged),
    [&](int) { update(); });
  QObject::connect(&qTglFill, &QCheckBox::toggled,
    [&](bool) { update(); });
  QObject::connect(&qBtnColorFill, &QPushButton::clicked,
    [&]() { qBtnColorFill.chooseColor(); update(); });
  QObject::connect(&qBtnBgColor1, &QPushButton::clicked,
    [&]() { qBtnBgColor1.chooseColor(); update(); });
  QObject::connect(&qBtnBgColor2, &QPushButton::clicked,
    [&]() { qBtnBgColor2.chooseColor(); update(); });
  // runtime loop
  update();
  return app.exec();
}

and a helper class qColorButton.h:

// borrowed from https://stackoverflow.com/a/55889624/7478597

#ifndef Q_COLOR_BUTTON_H
#define Q_COLOR_BUTTON_H

// Qt header:
#include <QColorDialog>
#include <QPushButton>

// a Qt push button for color selection
class QColorButton: public QPushButton {
  private:
    QColor _qColor;

  public:
    explicit QColorButton(
      const QString &text = QString(), const QColor &qColor = Qt::black,
      QWidget *pQParent = nullptr):
      QPushButton(text, pQParent)
    {
      setColor(qColor);
    }
    explicit QColorButton(
      const QColor &qColor = Qt::black,
      QWidget *pQParent = nullptr):
      QColorButton(QString(), qColor, pQParent)
    { }
    virtual ~QColorButton() = default;

    QColorButton(const QColorButton&) = delete;
    QColorButton& operator=(const QColorButton&) = delete;

    const QColor& color() const { return _qColor; }
    void setColor(const QColor &qColor)
    {
      _qColor = qColor;
      QFontMetrics qFontMetrics(font());
      const int h = qFontMetrics.height();
      QPixmap qPixmap(h, h);
      qPixmap.fill(_qColor);
      setIcon(qPixmap);
    }

    QColor chooseColor()
    {
      setColor(QColorDialog::getColor(_qColor, this, text()));
      return _qColor;
    }
};

#endif // Q_COLOR_BUTTON_H

When started, a default image is loaded and the simple matching is applied:

I downloaded the sample image from jloog.com/images/.

The result looks a bit poor.
The white background is matched but there appear white artefacts around black drawing.
This results from sampling where pixels which covered the drawing as well as the background got respective shades of gray.

So, a better approach is to turn the distance from pivot color into a respective alpha value whereby the threshold defines the range as well as a limit upto that colors shall be considered:

That looks better.

Now, I became curious how well this works in “real” photos:

The result is better when I was afraid.

However, it shows the limits of the approach I got so far.

Update:

While I was researching the web to get the precise conversion from RGB to HSV, I learnt a lot about the different HSL and HSV models I was not aware of before.
Finally, I stumbled into Color difference where I found some interesting statements:

As most definitions of color difference are distances within a color space, the standard means of determining distances is the Euclidean distance. If one presently has an RGB (Red, Green, Blue) tuple and wishes to find the color difference, computationally one of the easiest is to call R, G, B linear dimensions defining the color space.

…

There are a number of color distance formulae that attempt to use color spaces like HSV with the hue as a circle, placing the various colors within a three dimensional space of either a cylinder or cone, but most of these are just modifications of RGB; without accounting for differences in human color perception they will tend to be on par with a simple Euclidean metric.

So, I discarded the idea with matching in HSV space.

Instead, I made a very simple extension which IMHO provides a significant improvement concerning monochrom drawings:

The pixels with mixed drawing and background are transformed into shades of alpha but the RGB values are left untouched.
This is not quite correct because it should actually become the foreground color (pencil color) blended with alpha.
To fix this, I added an option to override the RGB values of the output with a color of choice.

This is the result with overridden color:

Btw. it allows a nice little extra effect – the pencil color can be modified:

(The above sample source code has been updated to reflect the last changes.)

To build the sample, either CMake can be used with this CMakeLists.txt:

project(QImageColorToAlpha)

cmake_minimum_required(VERSION 3.10.0)

set_property(GLOBAL PROPERTY USE_FOLDERS ON)
#set(CMAKE_CXX_STANDARD 17)
#set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)

find_package(Qt5Widgets CONFIG REQUIRED)

include_directories(
  "${CMAKE_SOURCE_DIR}")

add_executable(testQImageColorToAlpha
  testQImageColorToAlpha.cc
  qColorButton.h # qColorButton.cc
  imageColorToAlpha.h imageColorToAlpha.cc)

target_link_libraries(testQImageColorToAlpha
  Qt5::Widgets)

# define QT_NO_KEYWORDS to prevent confusion between of Qt signal-slots and
# other signal-slot APIs
target_compile_definitions(testQImageColorToAlpha PUBLIC QT_NO_KEYWORDS)

which I used to build it in VS2017.

Alternatively, a minimal Qt project file testQImageColorToAlpha.pro:

SOURCES = testQImageColorToAlpha.cc imageColorToAlpha.cc

QT += widgets

which I tested in cygwin:

$ qmake-qt5 testQImageColorToAlpha.pro 

$ make && ./testQImageColorToAlpha
g++ -c -fno-keep-inline-dllexport -D_GNU_SOURCE -pipe -O2 -Wall -W -D_REENTRANT -DQT_NO_DEBUG -DQT_WIDGETS_LIB -DQT_GUI_LIB -DQT_CORE_LIB -I. -isystem /usr/include/qt5 -isystem /usr/include/qt5/QtWidgets -isystem /usr/include/qt5/QtGui -isystem /usr/include/qt5/QtCore -I. -I/usr/lib/qt5/mkspecs/cygwin-g++ -o testQImageColorToAlpha.o testQImageColorToAlpha.cc
g++ -c -fno-keep-inline-dllexport -D_GNU_SOURCE -pipe -O2 -Wall -W -D_REENTRANT -DQT_NO_DEBUG -DQT_WIDGETS_LIB -DQT_GUI_LIB -DQT_CORE_LIB -I. -isystem /usr/include/qt5 -isystem /usr/include/qt5/QtWidgets -isystem /usr/include/qt5/QtGui -isystem /usr/include/qt5/QtCore -I. -I/usr/lib/qt5/mkspecs/cygwin-g++ -o imageColorToAlpha.o imageColorToAlpha.cc
g++  -o testQImageColorToAlpha.exe testQImageColorToAlpha.o imageColorToAlpha.o   -lQt5Widgets -lQt5Gui -lQt5Core -lGL -lpthread 
Qt Version: 5.9.4