ThermalExpert reverse engineering jar & dll

[frenky]

In the attachment are two versions of the same decompiled jar file which can be used as library for android app.
In them you can see USB communication and sensor data manipulation.
If code in one file looks strange, just look into other file where it might look more straightforward...

Interesting bits of code:

Get data from sensor

Code: [Select]

private int Read(byte[] buf, int iSize) {
            int iOffset = 0;
            if (iSize % 512 != 0) {
                iSize += 512 - iSize % 512;
            }
            int iRepeat = iSize / 16384;
            if (iSize % 16384 != 0) {
                ++iRepeat;
            }
            int iValue = iSize >> 9;
            try {
                if (this.m_USBConnection.controlTransfer(128, 134, iValue, 48879, this.m_VenBuf, 8, this.m_iReadTimeOut) < 0) {
                    return -1;
                }
            }
            catch (Exception e) {
                e.printStackTrace();
                this.m_Handler.obtainMessage(4, 0, 0, (Object)e.getLocalizedMessage()).sendToTarget();
                return -1;
            }
            int iRecvSize = 0;
            int iRecv = 0;
            ByteBuffer bf = ByteBuffer.wrap(buf);
            byte[] bufTemp = new byte[16384];
            try {
                int i = 0;
                do {
                    if (i >= iRepeat) {
                        return iOffset;
                    }
                    iRecvSize = Math.min(16384, iSize);
                    iRecv = this.m_USBConnection.bulkTransfer(this.m_USBRecvEndPoint, bufTemp, iRecvSize, this.m_iReadTimeOut);
                    if (iRecv != iRecvSize) {
                        return -1;
                    }
                    bf.put(bufTemp, 0, iRecvSize);
                    iOffset += iRecv;
                    iSize -= 16384;
                    ++i;
                } while (true);
            }
            catch (Exception e) {
                e.printStackTrace();
                this.m_Handler.obtainMessage(4, 0, 0, (Object)e.getLocalizedMessage()).sendToTarget();
                return iOffset;
            }
        }

Is actual sensor height 296? Data from sensor is 227,328 bytes which matches: 2byte * 384 * 296:

Code: [Select]

private final int m_fiQVGAPlusSensorWidth = 384;
        private final int m_fiQVGAPlusSensorHeight = 296;
        private final int m_fiQVGAPlusActiveHeight = 288;


[frenky]

Dead pixel correction seems quite similar to JadeW's code...

Code: [Select]

private void DeadCorrectionBeforeAGC() {
        int x;
        int cnt = 0;
        int sum = 0;
        int lower = - this.commData.getWindowingWidth();
        int upper = - lower;
        int right = 1;
        int left = -1;
        int y = 0;
        int pos = 0;
        while (y < this.commData.getWindowingHeight()) {
            x = 0;
            while (x < this.commData.getWindowingWidth()) {
                if (this.commData.m_pbyDeadData[pos] != 0) {
                    cnt = 0;
                    sum = 0;
                    if (y < this.commData.getWindowingHeight() - 1 && this.commData.m_pbyDeadData[pos + this.commData.getWindowingWidth()] == 0) {
                        sum += this.prevAGCData[pos + upper];
                        ++cnt;
                    }
                    if (y > 0 && this.commData.m_pbyDeadData[pos - this.commData.getWindowingWidth()] == 0) {
                        sum += this.prevAGCData[pos + lower];
                        ++cnt;
                    }
                    if (x > 0 && this.commData.m_pbyDeadData[pos - 1] == 0) {
                        sum += this.prevAGCData[pos + left];
                        ++cnt;
                    }
                    if (x < this.commData.getWindowingWidth() - 1 && this.commData.m_pbyDeadData[pos + 1] == 0) {
                        sum += this.prevAGCData[pos + right];
                        ++cnt;
                    }
                    this.prevAGCAfterDeadData[pos] = cnt > 0 ? (int)((double)(sum / cnt) + 0.5) : 0;
                } else {
                    this.prevAGCAfterDeadData[pos] = this.prevAGCData[pos];
                }
                ++x;
                ++pos;
            }
            ++y;
        }
        y = 0;
        pos = 0;
        while (y < this.commData.getWindowingHeight()) {
            x = 0;
            while (x < this.commData.getWindowingWidth()) {
                cnt = 0;
                sum = 0;
                if (this.commData.m_pb2DeadData[pos]) {
                    if (y < this.commData.getWindowingHeight() - 1 && !this.commData.m_pb2DeadData[pos + upper]) {
                        sum += this.prevAGCData[pos + upper];
                        ++cnt;
                    }
                    if (y > 0 && !this.commData.m_pb2DeadData[pos + lower]) {
                        sum += this.prevAGCData[pos + lower];
                        ++cnt;
                    }
                    if (x > 0 && !this.commData.m_pb2DeadData[pos + left]) {
                        sum += this.prevAGCData[pos + left];
                        ++cnt;
                    }
                    if (x < this.commData.getWindowingWidth() - 1 && !this.commData.m_pb2DeadData[pos + right]) {
                        sum += this.prevAGCData[pos + right];
                        ++cnt;
                    }
                    this.prevAGCAfterDeadData[pos] = cnt > 0 ? (int)((double)(sum / cnt) + 0.5) : 0;
                }
                ++x;
                ++pos;
            }
            ++y;
        }
        y = 0;
        pos = 0;
        while (y < this.commData.getWindowingHeight()) {
            x = 0;
            while (x < this.commData.getWindowingWidth()) {
                cnt = 0;
                sum = 0;
                if (this.commData.m_pb3DeadData[pos]) {
                    if (y < this.commData.getWindowingHeight() - 1 && !this.commData.m_pb3DeadData[pos + this.commData.getWindowingWidth()]) {
                        sum += this.prevAGCData[pos + upper];
                        ++cnt;
                    }
                    if (y > 0 && !this.commData.m_pb3DeadData[pos - this.commData.getWindowingWidth()]) {
                        sum += this.prevAGCData[pos + lower];
                        ++cnt;
                    }
                    if (x > 0 && !this.commData.m_pb3DeadData[pos - 1]) {
                        sum += this.prevAGCData[pos + left];
                        ++cnt;
                    }
                    if (x < this.commData.getWindowingWidth() - 1 && !this.commData.m_pb3DeadData[pos + 1]) {
                        sum += this.prevAGCData[pos + right];
                        ++cnt;
                    }
                    this.prevAGCAfterDeadData[pos] = cnt > 0 ? (int)((double)(sum / cnt) + 0.5) : 0;
                }
                ++x;
                ++pos;
            }
            ++y;
        }
        y = 0;
        pos = 0;
        while (y < this.commData.getWindowingHeight()) {
            x = 0;
            while (x < this.commData.getWindowingWidth()) {
                cnt = 0;
                sum = 0;
                if (this.commData.m_pb4DeadData[pos]) {
                    if (y < this.commData.getWindowingHeight() - 1 && !this.commData.m_pb4DeadData[pos + this.commData.getWindowingWidth()]) {
                        sum += this.prevAGCData[pos + upper];
                        ++cnt;
                    }
                    if (y > 0 && !this.commData.m_pb4DeadData[pos - this.commData.getWindowingWidth()]) {
                        sum += this.prevAGCData[pos + lower];
                        ++cnt;
                    }
                    if (x > 0 && !this.commData.m_pb4DeadData[pos - 1]) {
                        sum += this.prevAGCData[pos + left];
                        ++cnt;
                    }
                    if (x < this.commData.getWindowingWidth() - 1 && !this.commData.m_pb4DeadData[pos + 1]) {
                        sum += this.prevAGCData[pos + right];
                        ++cnt;
                    }
                    this.prevAGCAfterDeadData[pos] = cnt > 0 ? (int)((double)(sum / cnt) + 0.5) : 0;
                }
                ++x;
                ++pos;
            }
            ++y;
        }
    }

JadeW code:

Code: [Select]

void ThermalFrame::fixPixels(const std::vector<uint16_t> & pixels, bool use_given_pixel)
{
for (size_t i = 0; i < pixels.size(); ++i)
{
uint32_t pixel = pixels[i];

size_t x = pixel % 206;
size_t y = pixel / 206;


uint32_t val = use_given_pixel ? m_pixels[pixel] * 2 : 0;
uint8_t nr = use_given_pixel ? 2 : 0;

if (y > 0 && !m_bad_pixels[x][y - 1])
{
val += m_pixels[(y - 1) * 206 + x];
++nr;
}

if (y < 156 - 1 && !m_bad_pixels[x][y + 1])
{
val += m_pixels[(y + 1) * 206 + x];
++nr;
}

if (x > 0 && !m_bad_pixels[x - 1][y])
{
val += m_pixels[y * 206 + (x - 1)];
++nr;
}

if (x < 206 - 1 && !m_bad_pixels[x + 1][y])
{
val += m_pixels[y * 206 + x + 1];
++nr;
}

if (nr)
{
val /= nr;
m_pixels[pixel] = val;
}
else
m_pixels[pixel] = m_avg_val;
}
}


[frenky]

Sequence of first frames by joe-c:

https://www.eevblog.com/forum/thermal-imaging/i3-thermalexpert-custom-software/msg1055775/#msg1055775

[frenky]

T.E. Usb identifiers:
VID: 0x0547
PID: 0x0080

VID belongs to "Anchor Chips, Inc.".

[frenky]

Another interesting observation.

DLL call that captures single frame from sensor takes only 10ms (RecvImage or RecvImageDouble).
So sensor is capable of 100fps.
But If you make fast sequential calls you will notice approx 100ms delay between responses so that you can't go over 9fps...

[joe-c]

Is actual sensor height 296? Data from sensor is 227,328 bytes which matches: 2byte * 384 * 296:

No, just 384x288x2 but there additional lines with (yet) unknown data bytes.

i guess... Device Temperature, frame counter, Min and max values, Device serial... maybe other stuff...
i asked i3, but they wont tell... its "confidential" 

if you get a image over the DLL there is one frame transfered. but if you use the "CalcTemp" function with XY Pixel, i see no transfer over USB.
so i think this calculations (isAmbientCalibOn=true/false) will be done with the last received frame.

So each frame seems to be a full Frame with all you need for the Image. 

[frenky]

Interesting.   

I was investigating a bit DLL methods...

RecvImage returns values (257,514,771...) And if you divide every value by 257 you get values from 1 to 255. So you can directly use those values to output grayscale images.

RecvImageDouble return values approx like this: 30 -> 24*C; 210-> 36*C; 3700->183*C

So step of 23.7 in raw values is mapped to approx 1*C.

[joe-c]

RecvImage returns values (257,514,771...) And if you divide every value by 257 you get values from 1 to 255. So you can directly use those values to output grayscale images.

nice, i never noted before.

but i don't think that the Raw value directly can converted to a Temperature (without embedding the device temperature).
i see changes if the device heat up or cool down. like the seek, but far less strange.

[frenky]

One thing I noticed was that if you do calibration and then call RecvImageDouble, then the raw values "0" will represent calibration surface temperature.
(If my calibration plane is at 23*C then RecvImageDouble will return values >0 for temps >23 and  for will return raw <0 for temps <23).

I researched your conclusion that for calculating temp of a point it does not go to the sensor for reading...

On "ReadFlashData" it fills up two arrays. One is gain and another is offset.
When you request the temp of a point it calculates temperature from gain, offset and sensor reading. So there is no need to go to sensor for more data.

But this is also bad, because "ReadFlashData" happens only at the startup so sensor inner temp influences the measurements.
If you take a temp reading of a thermal plane at startup, let it run for 5min and then do the "ReadFlashData" again you will see that dll shows temperatures that are a few degrees less.

So for most stable readings you should let it run for 5min, do the "ReadFlashData" and then calibration.

[mahony]

I got my TEQ1+ last weekend and started playing a bit - very happy so far!

Not sure if this is still of interest, but here are my findings on this topic:
RecvImageDouble gives the image in some arbitrary scaled form of radiance per pixel with zero at the radiance seen on Shutter calibration (as already pointed out by frenky). What is nice (or not) is that you don't get a clue of integration times or gain settings, those are already 'in' those double 'pseudo radiance' values.
Usually a thermal imager should be more or less linear in DL vs. actuall irradiance in i.e. W/m²sr per integration time. The temperature then is derived from this radiance by some N-order polynom, which seems to be done in the DLLs, including FPA temperature according to the parts I dived into the decompiled jar files. (Thanks for that, this is very cool!)

If you do some calculations with Planck's formulas for the 8-14µm band and do a plot of radiance vs. temperature the result is very good approximated using a simple quadratic fit for the temperature range of the TE Q1. The other way around it might be a bit more tricky.

I captured a double image with objects from ~5°C up to ~100+°C and did the full temperature calculations and plotted a couple of relevant temperatures vs. the double values. As it turned out the double values from RecvImageDouble are almost perfecly linear with the expected Blackbody radiance values derived from the corresponding temperature readings.

What I now would like to do is diving a bit more into the jar files to get raw readings from sensor incl. FPA temp, integration times and or gain maybe and do a own 2 point NUC for further improved image quality... just need to find the time.

[frenky]

Nice, I'll be watching closely on your progress. 

[joe-c]

very interesting mahony.

i also make some trys with the TE and Planck values. But is use the WinUSB Mode (Zadig driver).
the results are good, but not perfect.

While regarding the Formula and change the values I was able to see, that the PlanckB and PlanckO seems to be the Key.
After change this 2 Values I was able let my TE Q1 see Temperatures.
But sadly I am not really known that I am doing. I searched for Planck values and in some Datasheets, but I don’t found a explanation how to acquire this. I just tried with known Values from a FLIR image and I see that the values are different on many Cameras from the same Type (FLIR E4).

See here a collection of FLIR E4 Calibration values:
https://www.eevblog.com/forum/thermal-imaging/flir-e4-thermal-imaging-camera-teardown/msg356616/#msg356616
there also changes for R1/R2. 4 parameters Change on the same Type of camera... that seems to be not a simple thing. 

Just for info, the soldering iron was from a digital station, set to 210°C.

[mahony]

Hi Joe-C,
very interesting.
I don't have the Excel sheet here but should be able to upload it when back home.

I wrote a quick and dirty piece of software based on the C# example code from fraser(?) ThexWinStream to display/stream and save files to disk. In one mode I captured a image via RecvImageDouble and did the full temperature calibration and save both files to be able to compare values pixel by pixel. The first thing that was obvious there is a significant blur after CalcTempertures, roughly gaussian like with a sigma of <1. I then computed radiance by integration of Plancks formula (from wikipedia: ) from 8 to 14µm for some significant temperatures. And those radiances are very linear with the double values from RecvImageDouble.

But there is already some calibration in those values. At least they do dead pixel removal, NUC and some compensation for integration time/read out gain.
I am not sure how i3 achieves its dynamic range. Maybe they use only one fixed integration time and just vary gain? Or the other way around, or both? I guess the transformation from 16bit DL to double values is due to the NUC, because the zero values correspond to the radiance/temp that was apparent from the calibration surface.

Another impressive thing I observed yesterday was the accuracy and dynamic range at different gain/t_int settings. I did a recording of the city and sky and very shortly sweep over the setting sun. The sun obviously saturated the image, but looking at the values for ground/houses and sky (~0°C to -30°C!), they didn't even change at all between frames with and without sun (except from noise)! Also the noise level in the low radiance/temp regions did not really change. Which leads me to the idea that the camera maybe always records with its full dynamic range (i.e. only one fixed gain/t_int) and only outputs the scaled frames.

I have to look a bit more into the details there 

edit/PS: I also had a look at your code using the Zadig driver, thanks for posting! Thats what I would also like to do further down the line and ideally get all the neccessarry calibration data from the camera if somehow possible (should be using the decompiled jar files  )

edit2: this link is quite good for online calculation of Planck-Intergrals: http://www.spectralcalc.com/blackbody_calculator/blackbody.php

[mahony]

I made a quick example with 'arbitrary' but realistic values for the temperatures and double readings:

[mahony]

As mentioned in the previous post: the Excel sheet and some images with and without sun in them (the Q1+ seems to max out around 280°C).
Both images show temperature reading at the exact same scale. There is no visual difference although in the image w/o sun the temperature range is only from -26°C to +8°C vs. +277°C with sun. Even when comparing the actual temperature reading at some significant spots, there is less than 0.5°C difference in both images. The same is true for the noise level in raw 'double' value images both with and without sun.

[tomas123]

edit2: this link is quite good for online calculation of Planck-Intergrals: http://www.spectralcalc.com/blackbody_calculator/blackbody.php

for information:
I posted here an Excel sheet for calculations of radiance:
https://www.eevblog.com/forum/thermal-imaging/yet-another-cheap-thermal-imager-incoming/msg779709/#msg779709

[mahony]

Thanks, that is quite handy!
I've wrote a small Tool in C# some years ago to do the calculations, also including a (very simple) atmospheric transmission lookup) - I should be able to upload it if someone is seeing use for it.

[joe-c]

Thermal Expert Planck Calibration
If the TE runs with WinUSB (zadig Driver), the temperature can be calculated as before with the 2 Point Cal. Since now it is additional possible to use a Planck Calculation, like on FLIR Images. Each Value can be changed and stored to file. This is a R&D Function with the goal, to make the TE to a self calibrated Thermal Camera with full access to the Parameters.
Now the Raw Device temperature is acquired.
Additional there 2 new Windows:
- TE Extra bytes (Tab "Ext 1"->"Extra Bytes")
shows the attached bytes after the pixel values. I found there some kind of device temperature map. Some unknown values are there... I guess offset 222759 looks like the minimal raw value repeated to end of frame
- Planc Calibration (Tab "Ext 2"->"Show P-Cal Window")
This Tool collect:
+ a Raw value (whole frame or if enabled, only average of cal box)
+ a reference value (you have to enter manual the real temperature of the object which result the raw value)
+ and the device Temperature (actual 2 Values for TEQ1 min and max)

If this 3 parameter was acquired (single click on "Get Cal Point"), there was 2 values calculated (the Temperature and the difference to the desired value) and the data appear in the table and the Graph (desired value green, calculated red, or blue for difference). The white colored entry's can be changed. The info entry holds normally the device temperature, but there you can also note something.
If you change one of the Panck values, the calculated Values change too and the result can be seen in the graph.
You can load an store cal files. Each file contains the constants, parameters and the Table values (not the calculated).
If you connect to the TE Camera with the "Ext 1" Tab, the file "TE_Cal_Autoload.TEC" was automatically loaded if exist.

Startup with your TEQ1 Camera:
1. Install Zadig Driver (should be shown as "I3SYTEM" in Device manager)
2. Start ThermoVision and go to Devices->Device: i3 T-Expert->Tab:Ext 1-> Connect
3. The button should be green, the button besides too, and there should "start" switch to "stop"
 (if not, unplug Camera from USB and reconnect... try again)
4. Click on "Create new GO-Maps" and point to a uniform heated cold surface (the button should be blue now)
5. Click on the same blue button and point to a uniform heated hot surface (the button should be red now)
6. Click on the same red button and it should go to gray.
7. Now point to uniform heated surface (the wall or close the lens cap) and click on "NUC" below the Tabs
Now you should have a valid map cal and a clean image.
Start the Planc Calibration (Tab "Ext 2"->"Show P-Cal Window") and acquire some raw values from know "thermal objects".
If you have some points over a wide range, you can start to see the results of the change from single planck values.
I can't provide default settings, I haven’t the time yet to make a setup with different values.

Add cal points: fist set the real temperature of your object and than point the camera to it (the cal window should be smaller than the Object) so you can click on "Get Cal Point" and the Raw value and Device Temperature was acquired automatically with the reference temperature you set before.
Remove cal points: Each row that has a marked cell would be removed if you right click with the mouse and click on "remove".

Version 1.4.3.2
Download: https://goo.gl/4OVlzs

I hope it work like expected, good luck
Joe-c

Edit:
some very useful links: 
https://www.eevblog.com/forum/thermal-imaging/flir-e4-thermal-imaging-camera-teardown/msg356616/#msg356616
(different calibration values for FLIR E4)
https://www.eevblog.com/forum/thermal-imaging/flir-e4-thermal-imaging-camera-teardown/msg348622/#msg348622
(self modify calibration FLIR E4)
http://u88.n24.queensu.ca/exiftool/forum/index.php/topic,4898.msg23944.html#msg23944
(FLIR Exif data and Temperature interpretation)
https://www.eevblog.com/forum/thermal-imaging/flir-e4-thermal-imaging-camera-teardown/msg342072/?topicseen#msg342072
(another useful link list from tomas123)

[mahony]

Wow!
Looks like you had a lot more time over the weekend than i did. 
Looking forward to try everything this evening.

There should be (the current?) FPA temperature around byte no 1531 in every frame (from FlashReadMultiOffset-method):

Code: [Select]


if (frameNum == 0) {
if (pos == 1518) {
this.shutterLess.m_dInitOnTimeCorrFactor = (double) Float.intBitsToFloat(convertInt);
tempCharPos2 = tempCharPos;
} else if (pos == 1520) {
r19 = this.shutterLess;
r0.m_pdInitOnTimeCorr[0] = (double) Float.intBitsToFloat(convertInt);
tempCharPos2 = tempCharPos;
} else if (pos == 1522) {
r19 = this.shutterLess;
r0.m_pdInitOnTimeCorr[1] = (double) Float.intBitsToFloat(convertInt);
tempCharPos2 = tempCharPos;
} else if (pos == 1524) {
r19 = this.shutterLess;
r0.m_pdInitOnTimeCorr[2] = (double) Float.intBitsToFloat(convertInt);
tempCharPos2 = tempCharPos;
} else if (pos == 1526) {
r19 = this.shutterLess;
r0.m_pdInitOnTimeCorr[3] = (double) Float.intBitsToFloat(convertInt);
tempCharPos2 = tempCharPos;
} else if (pos == 1528) {
r19 = this.shutterLess;
r0.m_pdInitOnTimeCorr[4] = (double) Float.intBitsToFloat(convertInt);
tempCharPos2 = tempCharPos;
} else if (pos == 1531) {
commData = this.commData;
r0.m_pdMultiOs_FpaTemp2[0][frameNum / 2] = (double) Float.intBitsToFloat(convertInt);
commData = this.commData;
r0.m_pdMultiOs_FpaTemp[frameNum / 2] = (double) Float.intBitsToFloat(convertInt);
tempCharPos2 = tempCharPos;
} else if (pos == 1533) {
this.shutterLess.setTemperatureOffset((double) Float.intBitsToFloat(convertInt));
tempCharPos2 = tempCharPos;
} else if (pos == 1535) {
this.shutterLess.setTemperatureGain((double) Float.intBitsToFloat(convertInt));
tempCharPos2 = tempCharPos;
}
} else if (frameNum % 2 == 0 && pos == 1531) {
commData = this.commData;
r0.m_pdMultiOs_FpaTemp2[0][frameNum / 2] = (double) Float.intBitsToFloat(convertInt);
commData = this.commData;
r0.m_pdMultiOs_FpaTemp[frameNum / 2] = (double) Float.intBitsToFloat(convertInt);
}

As far as I have seen there are also some calibration constants/values like temperature gain and offset in those first 4 lines of raw data (384*296*2 bytes) when using the Zadig/WinUSB drivers.
Didn't managed to dive much deeper into into it so far.

[joe-c]

Wow!
Looks like you had a lot more time over the weekend than i did. 

i don't had much time. it was enough for a release but not to make some Tests with my camera directly (and the calibration).

There should be (the current?) FPA temperature around byte no 1531 in every frame (from FlashReadMultiOffset-method):

really? not only in the cal frames at startup? i will look at this later. 

As far as I have seen there are also some calibration constants/values like temperature gain and offset in those first 4 lines of raw data (384*296*2 bytes) when using the Zadig/WinUSB drivers.
Didn't managed to dive much deeper into into it so far.

no.. after the frame the Temperatures come... i don't know why there so many values... i guess they have a temperature array to see if one corner of the chip has another temperature.
i read all values from this map, using this code:

Code: [Select]

for (int i = 221187; i < 222719; i+=2) {
ushort dTemp = (ushort)(frame.RawDataBytes[i]<<8|frame.RawDataBytes[i+1]);
if (DeviceTempRawMin>dTemp) { DeviceTempRawMin=dTemp; }
if (DeviceTempRawMax<dTemp) { DeviceTempRawMax=dTemp; }
}
and this is why the TE device temperature shows 2 values, the first is the lowest and the second the highest.
i see only little differences between both.

but i see an interesting thing... there is a glitch, each time the Raw Temperature  rise over 6910.
after 6913 it grow normal until the warmup is finished.

[mahony]

Hi Joe,
I had some time on monday to dive a little bit into the bits of the first few frames after startup that are used by the FlashRead method in the i3 DLL.
The first thing to notice is that in those first 27 frames (1+1+8+8+8+1), they have 4 leading 'data' lines (each 384*2 bytes) + 4 trailing, while all the real image frames (after frame 27) are having their data starting with no offset but show 8 trailing 'data' lines then.

I am now at the point where I know how to get data out of those first 26/27 frames:
FlashReadDead:
- reads the very first frame, there is a bad pixel map in the 'image' data - what I am not sure about is what the values are meant for. Good pixels have a value of 0, and then at least in my case, there are values of 1, 2 or 8 - I guess 4 might be in there as well? But not sure atm and I have to go to the decompiled code for more details.
FlashReadShutterless:
- read one 'dummy' frame, this is empty and ignore by the DLL although there are some bytes at the very beginning
- now the DLL is reading 8 consecutive frames to get Gain values: the interesting thing is the encoding of the data: There is some unusual reordering of the bytes followed by conversion to float (i.e. original byte order [0 1 2 3 4 5 6 7 8 ...] -> reordered [1 0 3 2 5 4 7 6 ...]). The result are 4 floating point values per pixel where the values for good pixels are roughly ~1.0/0.1/0.01/0.001 and quite far off at bad pixel positions. Those 4 values per pixels are stored consecutive in those 8 frames 'data' part.
- basically the same is done in the next 8 frames but now those are offset values. The byte-reordering is the same and also the 'pixel layout'. The values are ~7000.0/-20.0/0.1/0.002 for all pixels.
FlashReadMultiOffset:
- this one again reads 8 (MultiOsNum*2) frames. The same byte-reordering and float conversion as in FlashReadShutterless is used here. The result are again 4 values for each pixel but the pixel ordering is different. Each 2 consecutive frames  seem to form 1 frame of some value type where the first two frames show average values of roughly -6.0 while the next pairs are around 2.0/12.0/90.0 for my cam.
- there is also a lot of data in the top CDS line (the first 4) which is also usually encoded as above - but i did not get far with identifying the values and their usage

The same is true for all the other data. I guess the 4 gain and offset values are used for the basic calibration including correction for FPA temperature maybe - I have to find and understand the code consuming this data. The same with the MultiOffset data ... I need to find time to go further 

Ah and frame 27 seems to be some dummy frame again - at least for my cam there is a lot of bytes set but not obviously useful.

Getting closer ... 

[mahony]

I found the FPA temperature: in every image frame the actual image data is followed by 2 lines with values ~7000DL -> those are averaged and with this formula converted to the temperature in °C.

It happens at the beginning of display().
First those two lines are summed up and a count is build:

Code: [Select]

        int width = this.commData.getWindowingWidth(); //384
        int height = this.commData.getWindowingHeight(); //288
        int size = this.commData.getActiveSize(); // 384*288
        int tempPos = 0;
        for (int line = 0; line < 2; line += 1) { // only line 0 and 1 after actal image ! maybe this is out of actual active window?! -'> line isn't used?!
            int w = 0;
            while (w < this.commData.getSensorWidth()) { //column >= 3 && < sensorwidth! over 2 line ~700 values?!
                if (w >= 3) {
    // here all temperature values are summed up
                    tempPRD += (float) this.commData.m_piRecvData[size + tempPos]; // int[] m_piRecvData[384*288] -> 32bit raw image
                    tempPRDCount += 1; //keep track of count
                }
                w += 1;
                tempPos += 1;
            }
        }

and then the average is calculated and via a lengthy formula converted to temperatures:

Code: [Select]

        // averaged  FPAtemp
        if (tempPRDCount != 0) {
            // m_dFpaTemp = (333.3 * (((tempPRD / tempPRDCount * 4.5d) / 16384.0d) - 1.75d)) - 40.0d;
    this.shutterLess.m_dFpaTemp = (float) ((333.33333333333337d * (((((double) (tempPRD / ((float) tempPRDCount))) * 4.5d) / 16384.0d) - 1.75d)) - 40.0d);
        }

I have just recorded the 50 first frames after some time of running and the first values i see are ~7275 DL and on the last frame i get ~7280 DL, that corresponds to 42.71°C and 43.17°C using the above formula.
That sounds quite reasonable to me...

I also went through the full ReadFlashData method but I need to find where all the data is used and how, then I will put some more info together - and maybe some code. 

Good night...

[frenky]

Great job. 

[mahony]

Now we get to the interesting stuff. The following code is from method CalcTemp() in ThermalExpert-class.
This is where we end up when the temperature for some pixel location is to be calculated. Usually the way it goes is:
GetPointTemperature(int _x, int _y)
   find ambient correction temperature and call:
   -> PointTemperatur.GetPointTemp(double _ambientCorr)
                 -> CalcTemp(int _x, int _y) in ThermalExpert class!

Code: [Select]

    private double CalcTemp(int _x, int _y) {
        int imgWidth = this.commData.getWindowingWidth(); // 384
        int count = 0;
        double[] tempCalc = new double[9]; // temperature is calced/averaged over 3x3 patch!
        if (!this.shutterLess.m_bIsST) { // is set to true on shutterLess Ctr
            return 0.0d;
        }
        int i;
        double avgTemp = 0.0d;
// 3x3 patch around pixel of interest..
        for (int y = -1; y < 2; y += 1) {
            for (int x = -1; x < 2; x += 1) {
                double interGain = 0.0d;
                double interOs = 0.0d;
                int pos = ((y + _y) * imgWidth) + (x + _x);
                if (this.commData.m_pbyDeadData[pos] == null) {
// interGain and interOs as always... y=a+b*T+c*T²+d*T³
                    for (int order = 0; order < 4; order += 1) {
                        interGain += (double) (this.shutterLess.m_pdSL_FpaTempArray[order] * this.shutterLess.m_pdSL_Gain[(pos * 4) + order]);
                        interOs += (double) (this.shutterLess.m_pdSL_FpaTempArray[order] * this.shutterLess.m_pdSL_Offset[(pos * 4) + order]);
                    }
                    double dST_Gain = 256.0d / interGain;
                    double dST_Low = interOs;
                    double dST_High = dST_Gain + dST_Low;
                    this.shutterLess.getClass();
// 8192/256 * interGain * m_piRecvWinData[pos] = 32 * interGain * m_piRecvWinData[pos]!
                    double d = (8192.0d / dST_Gain) * ((double) this.commData.m_piRecvWinData[pos]); // int[] m_piRecvWinData = raw sensor data as 16 bits!
                    this.shutterLess.getClass();
// 16384 * (256/interGain + interOs)
                    double d2 = 16384.0d * dST_High;
                    this.shutterLess.getClass();
// d = 32*interGain*m_piRecvWinData[pos]
// d2 = 16384 * (256/interGain + interOs)
// dST_Low = interOs
// dST_Gain = 256/interGain
                    double dPixel = d + ((d2 - (24576.0d * dST_Low)) / dST_Gain); // dPixel = 32*[(m_piRecvWinData[pos] - interOs) * interGain + 512] ?!
                    tempCalc[count] = this.shutterLess.getTargetTemp(dPixel); //TBD: check in EXCEL!! -> yes this is correct
                    count += 1;
                }
            }
        }
        //removed the boring averaging stuff
...
    }

Input is commData.m_piRecvWinData[pos] which is an int[] of size [384*288] that holds the raw 16 bit detector values.

I think the comments explain the mechanism fairly well except from the part where 'interGain' and 'interOffset' are calculated.
Both values come from the initial FlashReadShutterless(). As stated in one of the previous posts the data from frame 3 ... 10 and 11 to 18 results in a gain map called shutterLess.m_pdSL_Gain[4*384*288] and shutterLess.m_pdSL_Offset[4*384*288] where for each image pixel there are 4 float with the order of magnitude [1.0 0.1 0.01 0.001] for the gain map and [7000 -20 0.1 0.002] for the offsets. For each pixel!

There is another array called shutterLess.m_pdSL_FpaTempArray[4] that hold 4 floats and is filled at the beginning of each call to display() (see earlier post). The values in there are of the form [1.0 T T² T³], where T is the current FPA temperature. Essentially what is in there ist FPAtemp^N with N=0...3.

The gain and offset values then are calculated with a cubic Polynom where the coefficients are the 4 values per pixel (c0 ... c3) and the variable it T (FPAtemp):
interGain = c0 + c1*T + c2*T² + c3*T³ ~= 1.0 + 0.1*T + 0.01*T² + 0.001*T³ or whatever the values for the pixel are

and respectively for the offset:
interOffset = a0 + a1*T + a2*T² + a3*T³  ....

The rest is pretty straight forward, altough quite complicated in code (not sure if this is due to some compiler optimizations or intended obfuscation?) but the formula breaks down to:
dPixel = 32*[(m_piRecvWinData[pos] - interOs) * interGain + 512]

Which delivers a double value for each pixel.
Finally this value is converted to a temperature via shutterLess.getTargetTemp(dPixel) which just applies a more or less simple formula as seen in code here:

Code: [Select]

public double getTargetTemp(double _output) {
if (169533.38422877376d < (7146.4357337d - _output) * 9.8736116d) {
return 0.0d;
}
// depending Gain and Offset values this may be in fact the calibration curve...
// m_dTemperatureGain = 1.0 initially -> set in FlashReadShutterlessMultiOffset() after temp offset ~1.25?
// m_dTemperatureOffset = 0.0 initially -> set in FlashReadShutterlessMultiOffset() directly after FPA temp ~3.5?
// m_dTestTempOffset = 0.0 initially -> set via setMaunalTemperatureOffset() but seems to called nowhere!!
// return (((Gain * (-411.74d + Math.sqrt(169533.38d - ((7146.43d - _output) * 9.87d)))) / 4.93d) - Offset) + manualOffset;
return (((this.m_dTemperatureGain * (-411.744319d + Math.sqrt(169533.38422877376d - ((7146.4357337d - _output) * 9.8736116d)))) / 4.9368058d) - this.m_dTemperatureOffset) + this.m_dTestTempOffset;
}

Et viola 16 bit raw data to temperature!

What is funny to mention is that for display there is a lot more going on with additional offsets. They are coming from FlashReadMultiOffset() in frames 19 ... 26 OR from the shutter calibration. But they seem to be unused when actually calculating temperatures.

The missing puzzle piece here is a bit earlier in GetPointTemp() of ThermalExpert class:

Code: [Select]

    public double GetPointTemperature(int _x, int _y) {
        double ambientCorrTemp = getAmbientCorrTemp(getAmbientTemp((double) this.shutterLess.m_dFpaTemp));
        this.m_pointTemp.SetPoint(convResolution(_x, _y));
        this.m_pointTemp.GetPoint();
        return this.m_pointTemp.GetPointTemp(ambientCorrTemp);
    }

getAmbientTemp:
The input again is the actual FPA temperature shutterLess.m_dFpaTemp.
commData.m_pdMultiOs_FpaTemp[4] are the 4 FPA temps form the init frames 19...26 at pixel position 1531 in FlashReadMultiOffset. In my case [51.21  33.84  62.63  43.38].

Code: [Select]

    private double getAmbientTemp(double _temp) {
        if (_temp >= this.commData.m_pdMultiOs_FpaTemp[1]) {
            return (this.ambientConstA[1] * _temp) + this.ambientConstB[1];
        }
        return (this.ambientConstA[0] * _temp) + this.ambientConstB[0];
    }

The ambientConstA/B are also calculated in FlashReadMultiOffset(). Those 4 FPA temperatures are sorted and then this is done:

Code: [Select]

            double[] multiChamberTemp = new double[3]; // this is fixed
            multiChamberTemp[0] = 5.0d;
            multiChamberTemp[1] = 25.0d;
            multiChamberTemp[2] = 35.0d;
            for (pos = 0; pos < 2; pos += 1) {
                if (sortFpaTemp[pos + 1] - sortFpaTemp[pos] == 0.0d) { // no in both positions (at least with my data, and any useful data i guess...)
                    this.ambientConstA[pos] = 1.0d; //this is only double[2] (all those arrays!)
                    this.ambientConstB[pos] = 0.0d;
                } else {
    // pos=0: = (25.0 - 5.0) / (43.38 - 33.84) = 2.0964
    // pos=1: = (35.0 - 25.0) / (51.21 - 43.38) = 1.2771
                    this.ambientConstA[pos] = (multiChamberTemp[pos + 1] - multiChamberTemp[pos]) / (sortFpaTemp[pos + 1] - sortFpaTemp[pos]);
                    // pos=0: = 5.0 - 2.0964*33.84 = -65.943
    // pos=1: = 25.0 - 1.2771*43.38 = -30.401
                    this.ambientConstB[pos] = multiChamberTemp[pos] - (this.ambientConstA[pos] * sortFpaTemp[pos]);
                }
            }
In my case the values should be:
A[0] = 2.0964     A[1] = 1.2771
B[0] = -65.943    B[1] = -30.401

So depending on which code path is executed and assuming and FPAtemp of 40°C getAmbientTemp should return:
2.09 * 40 - 65.94 = 17.66 or
1.27 * 40 - 30.40 = 20.40

getAmbientCorrTemp:
More than easy ... altough not really clear why and what happens.  Maybe some sort of estimation what intensity is received from the lens?

Code: [Select]

    private double getAmbientCorrTemp(double _ambientTemp) {
        return (-0.2d * _ambientTemp) + 5.0d;
    }
Using the values from above getAmbientCorrTemp will return:
-0.2 * 17.66 + 5.0 = 1.468 or
-0.2 * 20.40 + 5.0 = 0.920

And finally PointTemperature.GetPointTemp():
Which just calls the CalcTemp method shown in detail above and adds the ambient correction temperature.

Code: [Select]

        public double GetPointTemp(double _ambientCorr) {
            this.m_dTemp = ThermalExpert.this.CalcTemp(this.m_Point.x, this.m_Point.y) + _ambientCorr;
            return this.m_dTemp;
        }

Now that is really all I have for the moment ... 

If all goes well on the weekend I may be able to put some C# code together and try all this. The only issue I got so far is that I receive ReadTimeOut exceptions when querying the data from USB which resulted in a crash and loss of USB connection. I used the code from joe-c "TEQ1Thermal_003Maps" project to get my 50 frames for analysis. I tried a Thread.Sleep(100) between read request which resulted in exceptions after a couple of frames. To be save I raised it to Sleep(500) to just get a dataset to start with - that worked out. Anyone got any ideas on this or maybe 'save' working code?

Thanks and good night gents...

EDIT: there was an error in the calcualtions for the ambientCorr factors (used 0 as first temp instead of 5) - this is corrected now.

[joe-c]

great work mahony.

the temperature formula is interesting... i will show it deeper if i have more time.

I used the code from joe-c "TEQ1Thermal_003Maps" project to get my 50 frames for analysis. I tried a Thread.Sleep(100) between read request which resulted in exceptions after a couple of frames. To be save I raised it to Sleep(500) to just get a dataset to start with - that worked out. Anyone got any ideas on this or maybe 'save' working code?

Thread sleep? exceptions?
i tried it a few minutes ago with the 003Maps version... i have no problems with "Get 1 Frame ..." or Streaming.
could you tell me more?
i never noted a problem with "to fast readings from the TEQ1".

[mahony]

I basically just took the WinUSB class and the ThermalExpert class from this project and used it in a quick and dirty winforms app without any threading and so on that was put together in like 5 minutes.  Just click button, connect, read the first 50 frames from the TE and dump them to disk as raw binary data. Maybe the issue just sorts out when doing a proper setup with a separate thread to read from USB and pass the data on to where it is going to be processed.

I will double-check and get back to you. Your last ThermoVision worked like a charm so I am quite confident. 

 By the way: is it o.k. to PN or Email you if I am having further questions on this topic - could talk in german then 

[joe-c]

Sadly the formula doesn't woks for me yet.
With raw values 6994-6996 (collected 766 Values each frame) I get 17.08 and the expected Temperature was around 25° (with fresh connected Camera).
But I see your values are around 7275. I should make a measurement RawValue vs. Measured Case Temp later.

By the way: is it o.k. to PN or Email you if I am having further questions on this topic - could talk in german then 

Sure, everyone can contact me over email or PN. I guess the better way is email, because the PN don’t allow attachments.
Ansonsten auch gern in Deutsch, da versaue ich es nicht so oft mit der Grammatik 

Joe-c

[mahony]

Ansonsten auch gern in Deutsch, da versaue ich es nicht so oft mit der Grammatik 

Ja, da bin ich dabei ... 
I think for the forum it better to keep it english - they will forgive us our mistakes I think.

Hmm, strange for my values it produces absolutely reasonable results (see attached image) it was starting around 24°C which should be roughly room temperature and is climbing now - currently reaching ~29°C. Maybe they got different DLL versions?

I just check your code vs. mine and I am still not sure why i get the timeout in WinUSBDevice.ReadExactPipe ... with no 'sleeping' it happens after the first frame, with sleep of 200ms I still get this error after some 50-100 frames.
Using your code its totally fine... I will check this maybe tomorrow.

EDIT: I found/fixed my error WinUSBDevice.ReadExactPipe raised the exception when nothing could be read from the pipe, removed this exception and just return null, now everything works fine for the moment. Maybe later i set a different value for pipe read timeout.
 

[joe-c]

EDIT: I found/fixed my error WinUSBDevice.ReadExactPipe raised the exception when nothing could be read from the pipe, removed this exception and just return null, now everything works fine for the moment. Maybe later i set a different value for pipe read timeout.

well done. 

I tried a little bit with the warm-up of the Seek and the TE.
Next time I plan to use the collected "warm-up characterization".
I guess it should be possible to use the raw device temperature with a internal Look up table to get a offset for the raw value.
The Seek Thermal has much more drift while warm-up, if that LUT way works, the Seek can deliver usable temperatures while warm-up.

Btw... am I the only with the mystic glitch while warm-up of the TE?
 

[mahony]

Very nice comparision. Regarding the warm-up glitch: maybe this is due to the corrections that are applied. For display calculations the TE selects one of 4 offset maps (each corresponding to a factory set FPA temperature - read in ReadFlashMultiOffset). It selects the map that is closest to the current FPA temperature reading - so a switch between different offset maps may produce different average pixel values.

If you do a shutter calibration those factory offset table are ignored and the offsets derived from the shutter cal are used instead...

The most intersting thing to notice is that for calculation of temperatures non of those offsets is considered at all (see the lengthy post...) - this seems to be for display only.

I also had some time over the weekend to put some (very raw but working) code together. It will read all the neccessary data (gains, offsets, factory FPA temperatures and temp gain and offset) when the TE is first connected and store it to disk. If the software is shut down and restarted without disconnecting the TE the flash data isn't read from the cam but from disk instead... The pixels values are converted to the double values (as described earlier) and on click a temperature calculation is done over an 3x3 neighboorhood.
Additionally a cold and/or hot frame may be captured for 1 or 2 point correction of the available double valued images.

I am actually not totally sure anymore if the FPA temperature as calculated is actually are 'real' temperature in °C or some arbitrary scaled value, but I want to check this sometime this week with a cam at room temperature and doing a full 'warm-up run' as joe-c. Anyway it seems to work very good in the conversion of raw 16bit ints to doubles and further to temperatures as well.

Feel free to use modify or whatever the code. I compiled it with VS Community 2013 against .NET Framework 4.5 but it should be fine on older versions too (not the project files for sure). The code needs to be compiled with the 'allow unsafe' flag because the drawing routine (FastPainter class) uses some pointer stuff. There is no executable included atm. but I can attach it too if someone has use for it.

[tomas123]

@joe-c

nice charts 
I thought the seek did not have a shutter.

A year ago a posted here a warm-up chart from the Flir One:
https://www.eevblog.com/forum/thermal-imaging/flir-one-thermal-imaging-camera-teardown-and-hacks/msg848878/#msg848878

[mahony]

I did a 'warm-up run' yesterday using the posted code (with minor modifications for logging).
I will definetaly check my FPA temperature calculation again, because it was already starting ~43°C while it should have been around 25ish. I have already seen one mistake: in the DLL the used only values from row 3 an above in both 'temperature lines' - I just took one full line 

But anyway I post the charts and maybe update them some time later. What I captured was a room wall from ~1m distance with significant defocus because I also wanted to kind of characterize the noise over time. I captured/analyzed the frames converted to double values (so no raw DLs!). So in every frame I got I did some statistics: min, max, mean and standard deviation plus I did a calculation of temperature for a 3x3 pixel patch in the image center.

The first chart shows min, max and mean over roughly 55 minutes (scale is seconds) while the second shows the std deviation over the full frame.
After ~45 minutes i captured a cold frame (using the wall) and from there on it is using a 1 point correction. 4 to 5 minutes later i also did a hot frame (using my hand) and from there on it is doing the 2 point correction. You can see the significant drop in min/max and std-deviation after the cold frame.

The last chart shows the FPA temperture in red on the right scale (possibly wrong calculated) and the wall temperature from the center 3x3 pixels in blue vs. the left scaling.

After roughly 2-3 minutes all the readings become quite stable, altough the wall temperature still goes down a bit until the end. That might actually be true because one window in the room was partially open and it was in the evening so the wall may have cooled down a bit?

[joe-c]

I thought the seek did not have a shutter.

The TE has no shutter, but the Seek has. but it's a bad plastic shutter directly over the sensor. and the shutter rises pretty often (3-8 Seconds, depended from camera run time and warm-up).

After roughly 2-3 minutes all the readings become quite stable, altough the wall temperature still goes down a bit until the end. That might actually be true because one window in the room was partially open and it was in the evening so the wall may have cooled down a bit?

sounds true. I have the same problem with my wall.
I have to work on a stable temperature field... later.

I had a idea to compensate the warmup... just use a offset table.
I point the cold camera to the wall and start recording. I grab the raw device temperature and the frame avr (mean of all pixels).
this was stored in a WDC file (warmup drift correction).
then I use the raw value after warmup as reference and calculate the difference to all points before.
after connecting the cold seek again, this file was used to calculate a offset for each raw value in relation to the device raw value.
the results look interesting. but it works only if the camera is in a "calibrated temperature range".
and I use the normal mode, as you can see on the shutter drifts.

additional here is a graph from the TE. I made 2 times a 2 point map calibration, this is why the raw avr value has some break outs.

maybe I should faster think about a temperature stable calibration field to record the raw drift while the camera warm up to get a more accurate curve.

btw... thanks for your code sample mahony, I will look on it next time.

[mahony]

Nice plots again!  Interesting to see data from the Seek too. (always wanted to buy one - until I saw images from the TE... )

I didn't manage to do some more warm-up testing but couldn't stop thinking about the fact that the i3 DLL seems to be ignoring all the offsets from the flash and the shutter calibration when doing temperature calculations. So I did a quick test and took to image both in raw and temperature calculated mode (using i3 DLL mode) directly before and after a shutter calibration. I had the lens cap on to have a more or less uniform image. The result proved my interpretation of the code right (see attachments) - the raw double value image improved significantly after shutter calibration, while the temperatures calculated from those images did not change at all!

By the way: I am a big fan of the WinUSB mode now, partly because cam start-up is way faster now. When first connecting the cam I do the flash read which is faster than the DLL in itself (not sure what they are doing there?) and I also store all relevant data to disk. So on next startup (with out physically disconnecting the TE) the software just reads the flash data from disk, which gives immediate response. The other big pro (at least for me) is the fact that calculating a full temperature image is not blurring the image now. Plus my additional offset correction is also included in the temperature image = reduced noise and more homogeneous image without the blur when using the DLL.