Got gtest working and made a small example test

 file(GLOB SIP_GENERATED_SOURCE_FILES ${ISIS_SIP_CODE_DIR}/*.cpp)

 add_library(isispy MODULE ${SIP_GENERATED_SOURCE_FILES})

 target_link_libraries(isispy ${ALLLIBS} gtest ${CMAKE_THREAD_LIBS_INIT})

 target_link_libraries(isispy ${ALLLIBS})

 target_link_libraries(isispy isis3)

 set_target_properties(isispy PROPERTIES LINK_DEPENDS isis3 INSTALL_RPATH ${CMAKE_INSTALL_PREFIX})

 add_dependencies(isispy sipfiles)

  # - Base module depends on 3rd party libs, other libs also depend on base.

  # - Only the base module gets both a static and shared library.

  if(${name} STREQUAL ${CORE_LIB_NAME})

    set(reqLibs ${ALLLIBS})

    set(reqLibs "${ALLLIBS};gtest;${CMAKE_THREAD_LIBS_INIT}")

    set(alsoStatic ON)

  else()

    set(reqLibs "${CORE_LIB_NAME};${ALLLIBS}")

    set(reqLibs "${CORE_LIB_NAME};${ALLLIBS};gtest;${CMAKE_THREAD_LIBS_INIT}")

    set(alsoStatic OFF)

  endif()

cmake_minimum_required(VERSION 3.10)

add_dependencies(isis3 isis3)

# Link runISISTests with what we want to test and the GTest and pthread library

add_executable(runISISTests

               FileNameTests.cpp)

target_link_libraries(runISISTests isis3 ${ALLLIBS} ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES} pthread)

gtest_discover_tests(runISISTests WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/../tests)

#include "FileName.h"

#include <QString>

#include <gtest/gtest.h>

TEST(FileNameTests, BaseName) {

  QString test = "test.log";

  Isis::FileName file(test);

  EXPECT_EQ("test", file.baseName());

}

int main(int argc, char **argv) {

   ::testing::InitGoogleTest(&argc, argv);

   return RUN_ALL_TESTS();

}