# HG changeset patch
# User Bruno Haible <bruno@clisp.org>
# Date 1552604068 -3600
# Node ID e3496deebf303281a0b8c4616941212de1d5e4d5
# Parent  5a52ef2d47724a191333ea13d23cf33a8e37b405
isatty: Make it return true in Cygwin consoles on native Windows.

* lib/isatty.c: Include <string.h>.
(GetProcAddress): New macro.
(GetNamedPipeClientProcessIdFuncType): New type.
(GetNamedPipeClientProcessIdFunc): New variable.
(QueryFullProcessImageNameFuncType): New type.
(QueryFullProcessImageNameFunc): New variable.
(initialized): New variable.
(initialize): New function.
(IsCygwinConsoleHandle): New function.
(isatty): Invoke it.
* doc/posix-functions/isatty.texi: Mention the issue.

diff -r 5a52ef2d4772 -r e3496deebf30 ChangeLog
--- a/ChangeLog	Thu Mar 14 09:49:24 2019 +0100
+++ b/ChangeLog	Thu Mar 14 23:54:28 2019 +0100
@@ -1,3 +1,18 @@
+2019-03-14  Bruno Haible  <bruno@clisp.org>
+
+	isatty: Make it return true in Cygwin consoles on native Windows.
+	* lib/isatty.c: Include <string.h>.
+	(GetProcAddress): New macro.
+	(GetNamedPipeClientProcessIdFuncType): New type.
+	(GetNamedPipeClientProcessIdFunc): New variable.
+	(QueryFullProcessImageNameFuncType): New type.
+	(QueryFullProcessImageNameFunc): New variable.
+	(initialized): New variable.
+	(initialize): New function.
+	(IsCygwinConsoleHandle): New function.
+	(isatty): Invoke it.
+	* doc/posix-functions/isatty.texi: Mention the issue.
+
 2019-03-14  Bruno Haible  <bruno@clisp.org>
 
 	all: Update URLs to msdn.microsoft.com.
diff -r 5a52ef2d4772 -r e3496deebf30 doc/posix-functions/isatty.texi
--- a/doc/posix-functions/isatty.texi	Thu Mar 14 09:49:24 2019 +0100
+++ b/doc/posix-functions/isatty.texi	Thu Mar 14 23:54:28 2019 +0100
@@ -12,6 +12,8 @@
 On native Windows, this function also returns true for character devices such
 as @file{NUL}.
 @item
+On native Windows, this function returns false for Cygwin consoles.
+@item
 This function crashes when invoked with invalid arguments on some platforms:
 MSVC 14.
 @end itemize
diff -r 5a52ef2d4772 -r e3496deebf30 lib/isatty.c
--- a/lib/isatty.c	Thu Mar 14 09:49:24 2019 +0100
+++ b/lib/isatty.c	Thu Mar 14 23:54:28 2019 +0100
@@ -22,6 +22,7 @@
 /* This replacement is enabled on native Windows.  */
 
 #include <errno.h>
+#include <string.h>
 
 /* Get declarations of the Win32 API functions.  */
 #define WIN32_LEAN_AND_MEAN
@@ -38,12 +39,90 @@
 # include <io.h>
 #endif
 
+/* Avoid warnings from gcc -Wcast-function-type.  */
+#define GetProcAddress \
+  (void *) GetProcAddress
+
+/* GetNamedPipeClientProcessId was introduced only in Windows Vista.  */
+typedef BOOL (WINAPI * GetNamedPipeClientProcessIdFuncType) (HANDLE hPipe,
+                                                             PULONG pClientProcessId);
+static GetNamedPipeClientProcessIdFuncType GetNamedPipeClientProcessIdFunc = NULL;
+/* QueryFullProcessImageName was introduced only in Windows Vista.  */
+typedef BOOL (WINAPI * QueryFullProcessImageNameFuncType) (HANDLE hProcess,
+                                                           DWORD dwFlags,
+                                                           LPSTR lpExeName,
+                                                           PDWORD pdwSize);
+static QueryFullProcessImageNameFuncType QueryFullProcessImageNameFunc = NULL;
+static BOOL initialized = FALSE;
+
+static void
+initialize (void)
+{
+  HMODULE kernel32 = LoadLibrary ("kernel32.dll");
+  if (kernel32 != NULL)
+    {
+      GetNamedPipeClientProcessIdFunc =
+        (GetNamedPipeClientProcessIdFuncType) GetProcAddress (kernel32, "GetNamedPipeClientProcessId");
+      QueryFullProcessImageNameFunc =
+        (QueryFullProcessImageNameFuncType) GetProcAddress (kernel32, "QueryFullProcessImageNameA");
+    }
+  initialized = TRUE;
+}
+
 static BOOL IsConsoleHandle (HANDLE h)
 {
   DWORD mode;
+  /* GetConsoleMode
+     <https://docs.microsoft.com/en-us/windows/console/getconsolemode> */
   return GetConsoleMode (h, &mode) != 0;
 }
 
+static BOOL IsCygwinConsoleHandle (HANDLE h)
+{
+  /* A handle to a Cygwin console is in fact a named pipe whose client process
+     and server process is <CYGWIN_INSTALL_DIR>\bin\mintty.exe.  */
+  BOOL result = FALSE;
+  ULONG processId;
+
+  if (!initialized)
+    initialize ();
+
+  /* GetNamedPipeClientProcessId
+     <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-getnamedpipeclientprocessid>
+     It requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA or higher.  */
+  if (GetNamedPipeClientProcessIdFunc && QueryFullProcessImageNameFunc
+      && GetNamedPipeClientProcessIdFunc (h, &processId))
+    {
+      /* OpenProcess
+         <https://docs.microsoft.com/en-us/windows/desktop/api/processthreadsapi/nf-processthreadsapi-openprocess> */
+      HANDLE processHandle =
+        OpenProcess (PROCESS_QUERY_LIMITED_INFORMATION, FALSE, processId);
+      if (processHandle != NULL)
+        {
+          char buf[1024];
+          DWORD bufsize = sizeof (buf);
+          /* The file name can be determined through
+             GetProcessImageFileName
+             <https://docs.microsoft.com/en-us/windows/desktop/api/psapi/nf-psapi-getprocessimagefilenamea>
+             or
+             QueryFullProcessImageName
+             <https://docs.microsoft.com/en-us/windows/desktop/api/winbase/nf-winbase-queryfullprocessimagenamea>
+             The former returns a file name in non-standard notation (it starts
+             with '\Device\') and may require linking with psapi.dll.
+             The latter is better, but requires -D_WIN32_WINNT=_WIN32_WINNT_VISTA
+             or higher.  */
+          if (QueryFullProcessImageNameFunc (processHandle, 0, buf, &bufsize))
+            {
+              if (strlen (buf) >= 11
+                  && strcmp (buf + strlen (buf) - 11, "\\mintty.exe") == 0)
+                result = TRUE;
+            }
+          CloseHandle (processHandle);
+        }
+    }
+  return result;
+}
+
 #if HAVE_MSVC_INVALID_PARAMETER_HANDLER
 static int
 _isatty_nothrow (int fd)
@@ -84,6 +163,8 @@
       if (IsConsoleHandle (h))
         return 1;
     }
+  if (IsCygwinConsoleHandle (h))
+    return 1;
   errno = ENOTTY;
   return 0;
 }